diff --git a/src/Microsoft.Data.Analysis/DataFrame.BinaryOperators.tt b/src/Microsoft.Data.Analysis/DataFrame.BinaryOperators.tt index 52cbb913a7..f2c525b8f7 100644 --- a/src/Microsoft.Data.Analysis/DataFrame.BinaryOperators.tt +++ b/src/Microsoft.Data.Analysis/DataFrame.BinaryOperators.tt @@ -18,7 +18,7 @@ namespace Microsoft.Data.Analysis { public partial class DataFrame { - #pragma warning disable 1591 +#pragma warning disable 1591 <# foreach (MethodConfiguration method in methodConfiguration) { #> <# if (method.MethodType == MethodType.BinaryScalar) { #> <# if (method.IsBitwise == true) { #> @@ -32,7 +32,7 @@ namespace Microsoft.Data.Analysis { return df.<#=method.MethodName#>(value); } - + public static DataFrame operator <#=method.Operator#>(<#=type.TypeName#> value, DataFrame df) { return df.Reverse<#=method.MethodName#>(value); diff --git a/src/Microsoft.Data.Analysis/DataFrameColumn.BinaryOperators.tt b/src/Microsoft.Data.Analysis/DataFrameColumn.BinaryOperators.tt index aa68fe0d0f..e55c8249df 100644 --- a/src/Microsoft.Data.Analysis/DataFrameColumn.BinaryOperators.tt +++ b/src/Microsoft.Data.Analysis/DataFrameColumn.BinaryOperators.tt @@ -18,7 +18,7 @@ namespace Microsoft.Data.Analysis { public abstract partial class DataFrameColumn { - #pragma warning disable 1591 +#pragma warning disable 1591 <# foreach (MethodConfiguration method in methodConfiguration) { #> <# if (method.MethodType == MethodType.BinaryScalar) { #> <# if (method.IsBitwise == true) { #> diff --git a/src/Microsoft.Data.Analysis/PrimitiveColumnContainer.BinaryOperations.tt b/src/Microsoft.Data.Analysis/PrimitiveColumnContainer.BinaryOperations.tt index 1006508524..95fc73b176 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveColumnContainer.BinaryOperations.tt +++ b/src/Microsoft.Data.Analysis/PrimitiveColumnContainer.BinaryOperations.tt @@ -18,15 +18,15 @@ namespace Microsoft.Data.Analysis { <# foreach (MethodConfiguration method in methodConfiguration) { #> <# if (method.MethodType == MethodType.Comparison || method.MethodType == MethodType.ComparisonScalar) { #> - public <#= method.GetSingleArgumentMethodSignature("PrimitiveColumnContainer", "T") #> - { + public <#= method.GetSingleArgumentMethodSignature("PrimitiveColumnContainer", "T") #> + { <# if (method.MethodType == MethodType.ComparisonScalar ) { #> PrimitiveDataFrameColumnArithmetic.Instance.<#=method.MethodName#>(this, scalar, ret); <# } else { #> PrimitiveDataFrameColumnArithmetic.Instance.<#=method.MethodName#>(this, right, ret); <# } #> return this; - } + } <# } else { #> public <#= method.GetSingleArgumentMethodSignature("PrimitiveColumnContainer", "T")#> @@ -45,11 +45,11 @@ namespace Microsoft.Data.Analysis <# } #> <# foreach (MethodConfiguration method in methodConfiguration) { #> <# if (method.MethodType == MethodType.BinaryScalar) { #> - public PrimitiveColumnContainer Reverse<#=method.MethodName#>(T scalar) - { + public PrimitiveColumnContainer Reverse<#=method.MethodName#>(T scalar) + { PrimitiveDataFrameColumnArithmetic.Instance.<#=method.MethodName#>(scalar, this); return this; - } + } <# } #> <# } #> } diff --git a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumn.cs b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumn.cs index 3cd6799780..113b67cc1c 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumn.cs +++ b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumn.cs @@ -228,7 +228,7 @@ public override double Median() PrimitiveDataFrameColumn sortIndices = GetAscendingSortIndices(out Int64DataFrameColumn _); long middle = sortIndices.Length / 2; double middleValue = (double)Convert.ChangeType(this[sortIndices[middle].Value].Value, typeof(double)); - if (Length % 2 == 0) + if (sortIndices.Length % 2 == 0) { double otherMiddleValue = (double)Convert.ChangeType(this[sortIndices[middle - 1].Value].Value, typeof(double)); return (middleValue + otherMiddleValue) / 2; @@ -243,7 +243,7 @@ public override double Mean() { if (Length == 0) return 0; - return (double)Convert.ChangeType((T)Sum(), typeof(double)) / Length; + return (double)Convert.ChangeType((T)Sum(), typeof(double)) / (Length - NullCount); } protected internal override void Resize(long length) diff --git a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnComputations.cs b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnComputations.cs index a4c5d73b0d..dd1ad7542e 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnComputations.cs +++ b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnComputations.cs @@ -12,7 +12,7 @@ namespace Microsoft.Data.Analysis { internal interface IPrimitiveColumnComputation - where T : struct + where T : unmanaged { void Abs(PrimitiveColumnContainer column); void All(PrimitiveColumnContainer column, out bool ret); @@ -37,7 +37,7 @@ internal interface IPrimitiveColumnComputation } internal static class PrimitiveColumnComputation - where T : struct + where T : unmanaged { public static IPrimitiveColumnComputation Instance { get; } = PrimitiveColumnComputation.GetComputation(); } @@ -45,7 +45,7 @@ internal static class PrimitiveColumnComputation internal static class PrimitiveColumnComputation { public static IPrimitiveColumnComputation GetComputation() - where T : struct + where T : unmanaged { if (typeof(T) == typeof(bool)) { @@ -277,10 +277,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (byte)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (byte)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -336,10 +341,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (byte)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (byte)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -395,10 +405,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (byte)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (byte)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -454,10 +469,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (byte)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (byte)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -511,9 +531,14 @@ public void Max(PrimitiveColumnContainer column, out byte ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (byte)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (byte)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -548,9 +573,14 @@ public void Min(PrimitiveColumnContainer column, out byte ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (byte)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (byte)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -585,9 +615,14 @@ public void Product(PrimitiveColumnContainer column, out byte ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (byte)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (byte)(readOnlySpan[i] * ret); + } } } } @@ -622,9 +657,14 @@ public void Sum(PrimitiveColumnContainer column, out byte ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (byte)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (byte)(readOnlySpan[i] + ret); + } } } } @@ -704,10 +744,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (char)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (char)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -763,10 +808,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (char)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (char)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -822,10 +872,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (char)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (char)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -881,10 +936,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (char)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (char)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -938,9 +998,14 @@ public void Max(PrimitiveColumnContainer column, out char ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (char)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (char)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -975,9 +1040,14 @@ public void Min(PrimitiveColumnContainer column, out char ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (char)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (char)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -1012,9 +1082,14 @@ public void Product(PrimitiveColumnContainer column, out char ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (char)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (char)(readOnlySpan[i] * ret); + } } } } @@ -1049,9 +1124,14 @@ public void Sum(PrimitiveColumnContainer column, out char ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (char)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (char)(readOnlySpan[i] + ret); + } } } } @@ -1131,10 +1211,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (decimal)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (decimal)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -1190,10 +1275,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (decimal)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (decimal)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -1249,10 +1339,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (decimal)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (decimal)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -1308,10 +1403,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (decimal)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (decimal)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -1365,9 +1465,14 @@ public void Max(PrimitiveColumnContainer column, out decimal ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (decimal)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (decimal)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -1402,9 +1507,14 @@ public void Min(PrimitiveColumnContainer column, out decimal ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (decimal)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (decimal)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -1439,9 +1549,14 @@ public void Product(PrimitiveColumnContainer column, out decimal ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (decimal)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (decimal)(readOnlySpan[i] * ret); + } } } } @@ -1476,9 +1591,14 @@ public void Sum(PrimitiveColumnContainer column, out decimal ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (decimal)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (decimal)(readOnlySpan[i] + ret); + } } } } @@ -1558,10 +1678,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (double)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (double)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -1617,10 +1742,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (double)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (double)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -1676,10 +1806,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (double)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (double)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -1735,10 +1870,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (double)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (double)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -1792,9 +1932,14 @@ public void Max(PrimitiveColumnContainer column, out double ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (double)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (double)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -1829,9 +1974,14 @@ public void Min(PrimitiveColumnContainer column, out double ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (double)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (double)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -1866,9 +2016,14 @@ public void Product(PrimitiveColumnContainer column, out double ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (double)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (double)(readOnlySpan[i] * ret); + } } } } @@ -1903,9 +2058,14 @@ public void Sum(PrimitiveColumnContainer column, out double ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (double)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (double)(readOnlySpan[i] + ret); + } } } } @@ -1985,10 +2145,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (float)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (float)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2044,10 +2209,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (float)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (float)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2103,10 +2273,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (float)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (float)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2162,10 +2337,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (float)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (float)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2219,9 +2399,14 @@ public void Max(PrimitiveColumnContainer column, out float ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (float)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (float)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -2256,9 +2441,14 @@ public void Min(PrimitiveColumnContainer column, out float ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (float)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (float)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -2293,9 +2483,14 @@ public void Product(PrimitiveColumnContainer column, out float ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (float)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (float)(readOnlySpan[i] * ret); + } } } } @@ -2330,9 +2525,14 @@ public void Sum(PrimitiveColumnContainer column, out float ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (float)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (float)(readOnlySpan[i] + ret); + } } } } @@ -2412,10 +2612,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (int)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (int)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2471,10 +2676,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (int)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (int)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2530,10 +2740,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (int)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (int)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2589,10 +2804,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (int)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (int)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2646,9 +2866,14 @@ public void Max(PrimitiveColumnContainer column, out int ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (int)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (int)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -2683,9 +2908,14 @@ public void Min(PrimitiveColumnContainer column, out int ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (int)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (int)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -2720,9 +2950,14 @@ public void Product(PrimitiveColumnContainer column, out int ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (int)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (int)(readOnlySpan[i] * ret); + } } } } @@ -2757,9 +2992,14 @@ public void Sum(PrimitiveColumnContainer column, out int ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (int)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (int)(readOnlySpan[i] + ret); + } } } } @@ -2839,10 +3079,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (long)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (long)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2898,10 +3143,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (long)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (long)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -2957,10 +3207,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (long)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (long)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3016,10 +3271,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (long)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (long)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3073,9 +3333,14 @@ public void Max(PrimitiveColumnContainer column, out long ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (long)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (long)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -3110,9 +3375,14 @@ public void Min(PrimitiveColumnContainer column, out long ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (long)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (long)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -3147,9 +3417,14 @@ public void Product(PrimitiveColumnContainer column, out long ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (long)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (long)(readOnlySpan[i] * ret); + } } } } @@ -3184,9 +3459,14 @@ public void Sum(PrimitiveColumnContainer column, out long ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (long)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (long)(readOnlySpan[i] + ret); + } } } } @@ -3266,10 +3546,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (sbyte)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (sbyte)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3325,10 +3610,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (sbyte)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (sbyte)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3384,10 +3674,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (sbyte)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (sbyte)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3443,10 +3738,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (sbyte)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (sbyte)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3500,9 +3800,14 @@ public void Max(PrimitiveColumnContainer column, out sbyte ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (sbyte)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (sbyte)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -3537,9 +3842,14 @@ public void Min(PrimitiveColumnContainer column, out sbyte ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (sbyte)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (sbyte)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -3574,9 +3884,14 @@ public void Product(PrimitiveColumnContainer column, out sbyte ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (sbyte)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (sbyte)(readOnlySpan[i] * ret); + } } } } @@ -3611,9 +3926,14 @@ public void Sum(PrimitiveColumnContainer column, out sbyte ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (sbyte)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (sbyte)(readOnlySpan[i] + ret); + } } } } @@ -3693,10 +4013,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (short)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (short)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3752,10 +4077,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (short)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (short)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3811,10 +4141,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (short)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (short)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3870,10 +4205,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (short)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (short)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -3927,9 +4267,14 @@ public void Max(PrimitiveColumnContainer column, out short ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (short)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (short)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -3964,9 +4309,14 @@ public void Min(PrimitiveColumnContainer column, out short ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (short)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (short)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -4001,9 +4351,14 @@ public void Product(PrimitiveColumnContainer column, out short ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (short)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (short)(readOnlySpan[i] * ret); + } } } } @@ -4038,9 +4393,14 @@ public void Sum(PrimitiveColumnContainer column, out short ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (short)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (short)(readOnlySpan[i] + ret); + } } } } @@ -4120,10 +4480,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (uint)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (uint)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -4179,10 +4544,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (uint)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (uint)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -4238,10 +4608,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (uint)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (uint)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -4297,10 +4672,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (uint)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (uint)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -4354,9 +4734,14 @@ public void Max(PrimitiveColumnContainer column, out uint ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (uint)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (uint)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -4391,9 +4776,14 @@ public void Min(PrimitiveColumnContainer column, out uint ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (uint)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (uint)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -4428,9 +4818,14 @@ public void Product(PrimitiveColumnContainer column, out uint ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (uint)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (uint)(readOnlySpan[i] * ret); + } } } } @@ -4465,9 +4860,14 @@ public void Sum(PrimitiveColumnContainer column, out uint ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (uint)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (uint)(readOnlySpan[i] + ret); + } } } } @@ -4547,10 +4947,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ulong)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ulong)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -4606,10 +5011,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ulong)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ulong)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -4665,10 +5075,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ulong)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ulong)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -4724,10 +5139,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ulong)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ulong)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -4781,9 +5201,14 @@ public void Max(PrimitiveColumnContainer column, out ulong ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ulong)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ulong)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -4818,9 +5243,14 @@ public void Min(PrimitiveColumnContainer column, out ulong ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ulong)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ulong)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -4855,9 +5285,14 @@ public void Product(PrimitiveColumnContainer column, out ulong ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ulong)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ulong)(readOnlySpan[i] * ret); + } } } } @@ -4892,9 +5327,14 @@ public void Sum(PrimitiveColumnContainer column, out ulong ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ulong)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ulong)(readOnlySpan[i] + ret); + } } } } @@ -4974,10 +5414,15 @@ public void CumulativeMax(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ushort)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ushort)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -5033,10 +5478,15 @@ public void CumulativeMin(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ushort)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ushort)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -5092,10 +5542,15 @@ public void CumulativeProduct(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ushort)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ushort)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -5151,10 +5606,15 @@ public void CumulativeSum(PrimitiveColumnContainer column) var mutableBuffer = DataFrameBuffer.GetMutableBuffer(buffer); var mutableSpan = mutableBuffer.Span; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ushort)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ushort)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; + } } column.Buffers[b] = mutableBuffer; } @@ -5208,9 +5668,14 @@ public void Max(PrimitiveColumnContainer column, out ushort ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ushort)(Math.Max(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ushort)(Math.Max(readOnlySpan[i], ret)); + } } } } @@ -5245,9 +5710,14 @@ public void Min(PrimitiveColumnContainer column, out ushort ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ushort)(Math.Min(readOnlySpan[i], ret)); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ushort)(Math.Min(readOnlySpan[i], ret)); + } } } } @@ -5282,9 +5752,14 @@ public void Product(PrimitiveColumnContainer column, out ushort ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ushort)(readOnlySpan[i] * ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ushort)(readOnlySpan[i] * ret); + } } } } @@ -5319,9 +5794,14 @@ public void Sum(PrimitiveColumnContainer column, out ushort ret) { var buffer = column.Buffers[b]; var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { - ret = (ushort)(readOnlySpan[i] + ret); + if (column.IsValid(readOnlyBitMapSpan, i)) + { + ret = (ushort)(readOnlySpan[i] + ret); + } } } } diff --git a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnComputations.tt b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnComputations.tt index e71fc10a90..7749369ecc 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnComputations.tt +++ b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnComputations.tt @@ -19,7 +19,7 @@ using System.Collections.Generic; namespace Microsoft.Data.Analysis { internal interface IPrimitiveColumnComputation - where T : struct + where T : unmanaged { <# foreach (MethodConfiguration compMethod in computationMethodConfiguration) { #> <# if (compMethod.MethodType == MethodType.ElementwiseComputation && compMethod.HasReturnValue == false && compMethod.SupportsRowSubsets == true) {#> @@ -39,7 +39,7 @@ namespace Microsoft.Data.Analysis } internal static class PrimitiveColumnComputation - where T : struct + where T : unmanaged { public static IPrimitiveColumnComputation Instance { get; } = PrimitiveColumnComputation.GetComputation(); } @@ -47,7 +47,7 @@ namespace Microsoft.Data.Analysis internal static class PrimitiveColumnComputation { public static IPrimitiveColumnComputation GetComputation() - where T : struct + where T : unmanaged { <# foreach (TypeConfiguration type in typeConfiguration) { #> <#=GenerateIfStatementHeader(type)#> @@ -202,36 +202,41 @@ namespace Microsoft.Data.Analysis var mutableSpan = mutableBuffer.Span; <# } #> var readOnlySpan = buffer.ReadOnlySpan; + var readOnlyBitMapSpan = column.NullBitMapBuffers[b].ReadOnlySpan; + for (int i = 0; i < readOnlySpan.Length; i++) { + if (column.IsValid(readOnlyBitMapSpan, i)) + { <# if (method.MethodName == "CumulativeMax") { #> - ret = (<#=type.TypeName#>)(Math.Max(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + ret = (<#=type.TypeName#>)(Math.Max(readOnlySpan[i], ret)); + mutableSpan[i] = ret; <# } #> <# if (method.MethodName == "CumulativeMin") { #> - ret = (<#=type.TypeName#>)(Math.Min(readOnlySpan[i], ret)); - mutableSpan[i] = ret; + ret = (<#=type.TypeName#>)(Math.Min(readOnlySpan[i], ret)); + mutableSpan[i] = ret; <# } #> <# if (method.MethodName == "CumulativeProduct") { #> - ret = (<#=type.TypeName#>)(readOnlySpan[i] * ret); - mutableSpan[i] = ret; + ret = (<#=type.TypeName#>)(readOnlySpan[i] * ret); + mutableSpan[i] = ret; <# } #> <# if (method.MethodName == "CumulativeSum") { #> - ret = (<#=type.TypeName#>)(readOnlySpan[i] + ret); - mutableSpan[i] = ret; + ret = (<#=type.TypeName#>)(readOnlySpan[i] + ret); + mutableSpan[i] = ret; <# } #> <# if (method.MethodName == "Max") { #> - ret = (<#=type.TypeName#>)(Math.Max(readOnlySpan[i], ret)); + ret = (<#=type.TypeName#>)(Math.Max(readOnlySpan[i], ret)); <# } #> <# if (method.MethodName == "Min") { #> - ret = (<#=type.TypeName#>)(Math.Min(readOnlySpan[i], ret)); + ret = (<#=type.TypeName#>)(Math.Min(readOnlySpan[i], ret)); <# } #> <# if (method.MethodName == "Product") { #> - ret = (<#=type.TypeName#>)(readOnlySpan[i] * ret); + ret = (<#=type.TypeName#>)(readOnlySpan[i] * ret); <# } #> <# if (method.MethodName == "Sum") { #> - ret = (<#=type.TypeName#>)(readOnlySpan[i] + ret); + ret = (<#=type.TypeName#>)(readOnlySpan[i] + ret); <# } #> + } } <# if (method.MethodName == "CumulativeMax" || method.MethodName == "CumulativeMin" || method.MethodName == "CumulativeProduct" || method.MethodName == "CumulativeSum") { #> column.Buffers[b] = mutableBuffer; diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs index 06286b3ae4..1695ded747 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs @@ -844,7 +844,7 @@ public void TestComputations() Assert.Equal(100.0, df.Columns["Double"].Max()); Assert.Equal(-10.0f, df.Columns["Float"].Min()); Assert.Equal((uint)0, df.Columns["Uint"].Product()); - Assert.Equal((ushort)140, df.Columns["Ushort"].Sum()); + Assert.Equal((ushort)130, df.Columns["Ushort"].Sum()); df.Columns["Double"][0] = 100.1; Assert.Equal(100.1, df.Columns["Double"][0]); @@ -1013,7 +1013,7 @@ public void TestComputationsIncludingDateTime() Assert.Equal(100.0, df.Columns["Double"].Max()); Assert.Equal(-10.0f, df.Columns["Float"].Min()); Assert.Equal((uint)0, df.Columns["Uint"].Product()); - Assert.Equal((ushort)140, df.Columns["Ushort"].Sum()); + Assert.Equal((ushort)130, df.Columns["Ushort"].Sum()); df.Columns["Double"][0] = 100.1; Assert.Equal(100.1, df.Columns["Double"][0]); @@ -3323,5 +3323,15 @@ public void GetColumnTests() Assert.Throws(() => dataFrame.Columns.GetSingleColumn("Ushort")); } + + [Fact] + public void TestMeanMedian() + { + DataFrame df = MakeDataFrameWithNumericColumns(10, true, 0); + + Assert.Equal(40.0 / 9.0, df["Decimal"].Mean()); + Assert.Equal(4, df["Decimal"].Median()); + + } } } diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj index 872769ef74..1a570416e4 100644 --- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj +++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj @@ -16,6 +16,10 @@ TextTemplatingFileGenerator DataFrameColumn.BinaryOperationTests.cs + + TextTemplatingFileGenerator + PrimitiveDataFrameColumnComputationsTests.cs + @@ -28,11 +32,16 @@ True DataFrameColumn.BinaryOperationTests.tt + + True + True + PrimitiveDataFrameColumnComputationsTests.tt + - + diff --git a/test/Microsoft.Data.Analysis.Tests/PrimitiveDataFrameColumnComputationsTests.cs b/test/Microsoft.Data.Analysis.Tests/PrimitiveDataFrameColumnComputationsTests.cs new file mode 100644 index 0000000000..476c51c5b3 --- /dev/null +++ b/test/Microsoft.Data.Analysis.Tests/PrimitiveDataFrameColumnComputationsTests.cs @@ -0,0 +1,157 @@ + + +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +// Generated from PrimitiveDataFrameColumnComputationsTests.tt. Do not modify directly + + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Microsoft.Data.Analysis.Tests +{ + public partial class DataFrameColumnTests + { + IEnumerable ByteValues = new byte?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable CharValues = new char?[] { (char)1, null, (char)2, (char)3, (char)4, null, (char)6, (char)7 }; + IEnumerable DecimalValues = new decimal?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable DoubleValues = new double?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable SingleValues = new float?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable Int32Values = new int?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable Int64Values = new long?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable SByteValues = new sbyte?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable Int16Values = new short?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable UInt32Values = new uint?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable UInt64Values = new ulong?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable UInt16Values = new ushort?[] { 1, null, 2, 3, 4, null, 6, 7 }; + + + [Fact] + public void ByteColumnComputationsTests() + { + + var column = new ByteDataFrameColumn("byteValues", ByteValues); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void CharColumnComputationsTests() + { + + var column = new CharDataFrameColumn("charValues", CharValues); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void DecimalColumnComputationsTests() + { + + var column = new DecimalDataFrameColumn("decimalValues", DecimalValues); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void DoubleColumnComputationsTests() + { + + var column = new DoubleDataFrameColumn("doubleValues", DoubleValues); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void SingleColumnComputationsTests() + { + + var column = new SingleDataFrameColumn("floatValues", SingleValues); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void Int32ColumnComputationsTests() + { + + var column = new Int32DataFrameColumn("intValues", Int32Values); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void Int64ColumnComputationsTests() + { + + var column = new Int64DataFrameColumn("longValues", Int64Values); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void SByteColumnComputationsTests() + { + + var column = new SByteDataFrameColumn("sbyteValues", SByteValues); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void Int16ColumnComputationsTests() + { + + var column = new Int16DataFrameColumn("shortValues", Int16Values); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void UInt32ColumnComputationsTests() + { + + var column = new UInt32DataFrameColumn("uintValues", UInt32Values); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void UInt64ColumnComputationsTests() + { + + var column = new UInt64DataFrameColumn("ulongValues", UInt64Values); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + [Fact] + public void UInt16ColumnComputationsTests() + { + + var column = new UInt16DataFrameColumn("ushortValues", UInt16Values); + + Assert.Equal(Enumerable.Max(Int32Values), Convert.ToInt32(column.Max())); + Assert.Equal(Enumerable.Min(Int32Values), Convert.ToInt32(column.Min())); + Assert.Equal(Enumerable.Sum(Int32Values), Convert.ToInt32(column.Sum())); + } + } +} + + + diff --git a/test/Microsoft.Data.Analysis.Tests/PrimitiveDataFrameColumnComputationsTests.tt b/test/Microsoft.Data.Analysis.Tests/PrimitiveDataFrameColumnComputationsTests.tt new file mode 100644 index 0000000000..df9e465358 --- /dev/null +++ b/test/Microsoft.Data.Analysis.Tests/PrimitiveDataFrameColumnComputationsTests.tt @@ -0,0 +1,73 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Text" #> +<#@ import namespace="System.Collections.Generic" #> +<#@ output extension=".cs" #> +<#@ include file="..\..\src\Microsoft.Data.Analysis\ColumnArithmeticTemplate.ttinclude"#> +<#@ include file="..\..\src\Microsoft.Data.Analysis\PrimitiveDataFrameColumn.BinaryOperations.Combinations.ttinclude" #> +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +// Generated from PrimitiveDataFrameColumnComputationsTests.tt. Do not modify directly + + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Microsoft.Data.Analysis.Tests +{ + public partial class DataFrameColumnTests + { + IEnumerable ByteValues = new byte?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable CharValues = new char?[] { (char)1, null, (char)2, (char)3, (char)4, null, (char)6, (char)7 }; + IEnumerable DecimalValues = new decimal?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable DoubleValues = new double?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable SingleValues = new float?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable Int32Values = new int?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable Int64Values = new long?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable SByteValues = new sbyte?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable Int16Values = new short?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable UInt32Values = new uint?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable UInt64Values = new ulong?[] { 1, null, 2, 3, 4, null, 6, 7 }; + IEnumerable UInt16Values = new ushort?[] { 1, null, 2, 3, 4, null, 6, 7 }; + + +<# + foreach (TypeConfiguration type in typeConfiguration) + { + if (type.TypeName == "bool" || type.TypeName == "DateTime") continue; +#> + [Fact] + public void <#=GetCapitalizedPrimitiveTypes(type.TypeName)#>ColumnComputationsTests() + { + + var column = new <#=GetCapitalizedPrimitiveTypes(type.TypeName)#>DataFrameColumn("<#=type.TypeName#>Values", <#=GetCapitalizedPrimitiveTypes(type.TypeName)#>Values); + +<# + foreach (MethodConfiguration method in computationMethodConfiguration) + { +#> +<# if (method.MethodType == MethodType.Reduction && method.MethodName != "Product" && method.IsNumeric && type.SupportsNumeric && !type.UnsupportedMethods.Contains(method.MethodName) ) { + + if(method.SupportsRowSubsets){ +#> +<# + } + else + { +#> + Assert.Equal(Enumerable.<#=method.MethodName#>(Int32Values), Convert.ToInt32(column.<#=method.MethodName#>())); +<# } #> +<# } #> +<# } #> + } +<# } #> + } +} + + +