< Summary

Information
Class: System.Text.Ascii
Assembly: System.Private.CoreLib
File(s): File 1: https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.CaseConversion.cs
File 2: https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.cs
File 3: https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs
File 4: https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Transcoding.cs
File 5: https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Trimming.cs
File 6: https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs
File 7: https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.Helpers.cs
Line coverage
41%
Covered lines: 452
Uncovered lines: 635
Coverable lines: 1087
Total lines: 3928
Line coverage: 41.5%
Branch coverage
31%
Covered branches: 243
Total branches: 766
Branch coverage: 31.7%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Metrics

MethodBranch coverage Cyclomatic complexity NPath complexity Sequence coverage
File 1: ToUpper(...)100%110%
File 1: ToUpper(...)100%110%
File 1: ToUpper(...)100%110%
File 1: ToUpper(...)100%110%
File 1: ToLower(...)100%110%
File 1: ToLower(...)100%11100%
File 1: ToLower(...)100%110%
File 1: ToLower(...)100%110%
File 1: ToLowerInPlace(...)100%110%
File 1: ToLowerInPlace(...)100%110%
File 1: ToUpperInPlace(...)100%110%
File 1: ToUpperInPlace(...)100%110%
File 1: ChangeCase(...)50%6676.92%
File 1: ChangeCase(...)0%220%
File 1: ChangeCase(...)20.83%727239.77%
File 1: ChangeWidthAndWriteTo(...)8.33%121216.66%
File 1: SignedLessThan(...)50%4460%
File 2: IsValid(...)100%110%
File 2: IsValid(...)100%11100%
File 2: IsValid(...)100%110%
File 2: IsValid(...)100%110%
File 2: IsValidCore(...)10.41%484810.22%
File 3: Equals(...)0%220%
File 3: Equals(...)0%220%
File 3: Equals(...)100%110%
File 3: Equals(...)0%220%
File 3: Equals(...)0%60600%
File 3: EqualsIgnoreCase(...)0%220%
File 3: EqualsIgnoreCase(...)0%220%
File 3: EqualsIgnoreCase(...)100%110%
File 3: EqualsIgnoreCase(...)0%220%
File 3: EqualsIgnoreCase(...)100%110%
File 3: EqualsIgnoreCase(...)0%92920%
File 3: Load128(...)100%110%
File 3: Load256(...)100%110%
File 3: Load512(...)100%110%
File 3: EqualAndAscii256(...)0%440%
File 3: EqualAndAscii512(...)0%440%
File 3: Load128(...)0%660%
File 3: Load256(...)100%110%
File 3: Load512(...)100%110%
File 3: EqualAndAscii256(...)0%440%
File 3: EqualAndAscii512(...)0%440%
File 4: ToUtf16(...)75%4481.81%
File 4: FromUtf16(...)0%440%
File 5: Trim(...)100%110%
File 5: Trim(...)100%110%
File 5: TrimStart(...)100%110%
File 5: TrimStart(...)100%110%
File 5: TrimEnd(...)100%110%
File 5: TrimEnd(...)100%110%
File 5: TrimHelper(...)0%16160%
File 6: AllBytesInUInt64AreAscii(...)100%110%
File 6: AllCharsInUInt32AreAscii(...)100%11100%
File 6: AllCharsInUInt64AreAscii(...)50%44100%
File 6: GetIndexOfFirstNonAsciiByte(...)37.5%8880%
File 6: GetIndexOfFirstNonAsciiByte_Vector(...)92.85%5656100%
File 6: ContainsNonAsciiByte_Sse2(...)100%110%
File 6: GetIndexOfFirstNonAsciiByte_Intrinsified(...)0%68680%
File 6: GetIndexOfFirstNonAsciiChar(...)37.5%8880%
File 6: GetIndexOfFirstNonAsciiChar_Vector(...)55.76%525268.75%
File 6: GetIndexOfFirstNonAsciiChar_Intrinsified(...)0%46460%
File 6: NarrowFourUtf16CharsToAsciiAndWriteToBuffer(...)25%4433.33%
File 6: NarrowTwoUtf16CharsToAsciiAndWriteToBuffer(...)100%11100%
File 6: NarrowUtf16ToAscii(...)97.05%3434100%
File 6: VectorContainsNonAsciiChar(...)25%4450%
File 6: VectorContainsNonAsciiChar(...)16.66%6628.57%
File 6: VectorContainsNonAsciiChar(...)100%11100%
File 6: VectorContainsNonAsciiChar(...)100%11100%
File 6: VectorContainsNonAsciiChar(...)50%44100%
File 6: AllCharsInVectorAreAscii(...)0%660%
File 6: AllCharsInVectorAreAscii(...)0%660%
File 6: AllCharsInVectorAreAscii(...)0%440%
File 6: ExtractAsciiVector(...)16.66%6650%
File 6: ExtractAsciiVector(...)50%22100%
File 6: ExtractAsciiVector(...)50%22100%
File 6: NarrowUtf16ToAscii_Intrinsified(...)92.85%1414100%
File 6: NarrowUtf16ToAscii_Intrinsified_256(...)92.85%1414100%
File 6: NarrowUtf16ToAscii_Intrinsified_512(...)92.85%1414100%
File 6: WidenAsciiToUtf16(...)91.17%3434100%
File 6: WidenAsciiToUtf1_Vector(...)87.5%8895.45%
File 6: HasMatch(...)100%11100%
File 6: Widen(...)100%44100%
File 6: WidenFourAsciiBytesToUtf16AndWriteToBuffer(...)75%4433.33%
File 7: AllBytesInUInt32AreAscii(...)100%11100%
File 7: CountNumberOfLeadingAsciiBytesFromUInt32WithSomeNonAsciiData(...)100%11100%

File(s)

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.CaseConversion.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Buffers;
 5using System.Diagnostics;
 6using System.Diagnostics.CodeAnalysis;
 7using System.Numerics;
 8using System.Runtime.CompilerServices;
 9using System.Runtime.InteropServices;
 10using System.Runtime.Intrinsics;
 11using System.Runtime.Intrinsics.X86;
 12using System.Text.Unicode;
 13
 14namespace System.Text
 15{
 16    public static partial class Ascii
 17    {
 18        /// <summary>
 19        /// Copies text from a source buffer to a destination buffer, converting
 20        /// ASCII letters to uppercase during the copy.
 21        /// </summary>
 22        /// <param name="source">The source buffer from which ASCII text is read.</param>
 23        /// <param name="destination">The destination buffer to which uppercase text is written.</param>
 24        /// <param name="bytesWritten">The number of bytes actually written to <paramref name="destination"/>. It's the 
 25        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 26        /// <remarks>In-place conversion is prohibited, please use <see cref="ToUpperInPlace(Span{byte}, out int)"/> for
 27        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 28        public static OperationStatus ToUpper(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesWritten)
 029            => ChangeCase<byte, byte, ToUpperConversion>(source, destination, out bytesWritten);
 30
 31        /// <summary>
 32        /// Copies text from a source buffer to a destination buffer, converting
 33        /// ASCII letters to uppercase during the copy.
 34        /// </summary>
 35        /// <param name="source">The source buffer from which ASCII text is read.</param>
 36        /// <param name="destination">The destination buffer to which uppercase text is written.</param>
 37        /// <param name="charsWritten">The number of characters actually written to <paramref name="destination"/>. It's
 38        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 39        /// <remarks>In-place conversion is prohibited, please use <see cref="ToUpperInPlace(Span{char}, out int)"/> for
 40        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 41        public static OperationStatus ToUpper(ReadOnlySpan<char> source, Span<char> destination, out int charsWritten)
 042            => ChangeCase<ushort, ushort, ToUpperConversion>(MemoryMarshal.Cast<char, ushort>(source), MemoryMarshal.Cas
 43
 44        /// <summary>
 45        /// Copies text from a source buffer to a destination buffer, converting
 46        /// ASCII letters to uppercase during the copy.
 47        /// </summary>
 48        /// <param name="source">The source buffer from which ASCII text is read.</param>
 49        /// <param name="destination">The destination buffer to which uppercase text is written.</param>
 50        /// <param name="charsWritten">The number of characters actually written to <paramref name="destination"/>. It's
 51        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 52        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 53        public static OperationStatus ToUpper(ReadOnlySpan<byte> source, Span<char> destination, out int charsWritten)
 054            => ChangeCase<byte, ushort, ToUpperConversion>(source, MemoryMarshal.Cast<char, ushort>(destination), out ch
 55
 56        /// <summary>
 57        /// Copies text from a source buffer to a destination buffer, converting
 58        /// ASCII letters to uppercase during the copy.
 59        /// </summary>
 60        /// <param name="source">The source buffer from which ASCII text is read.</param>
 61        /// <param name="destination">The destination buffer to which uppercase text is written.</param>
 62        /// <param name="bytesWritten">The number of bytes actually written to <paramref name="destination"/>. It's the 
 63        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 64        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 65        public static OperationStatus ToUpper(ReadOnlySpan<char> source, Span<byte> destination, out int bytesWritten)
 066            => ChangeCase<ushort, byte, ToUpperConversion>(MemoryMarshal.Cast<char, ushort>(source), destination, out by
 67
 68        /// <summary>
 69        /// Copies text from a source buffer to a destination buffer, converting
 70        /// ASCII letters to lowercase during the copy.
 71        /// </summary>
 72        /// <param name="source">The source buffer from which ASCII text is read.</param>
 73        /// <param name="destination">The destination buffer to which lowercase text is written.</param>
 74        /// <param name="bytesWritten">The number of bytes actually written to <paramref name="destination"/>. It's the 
 75        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 76        /// <remarks>In-place conversion is prohibited, please use <see cref="ToLowerInPlace(Span{byte}, out int)"/> for
 77        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 78        public static OperationStatus ToLower(ReadOnlySpan<byte> source, Span<byte> destination, out int bytesWritten)
 079            => ChangeCase<byte, byte, ToLowerConversion>(source, destination, out bytesWritten);
 80
 81        /// <summary>
 82        /// Copies text from a source buffer to a destination buffer, converting
 83        /// ASCII letters to lowercase during the copy.
 84        /// </summary>
 85        /// <param name="source">The source buffer from which ASCII text is read.</param>
 86        /// <param name="destination">The destination buffer to which lowercase text is written.</param>
 87        /// <param name="charsWritten">The number of characters actually written to <paramref name="destination"/>. It's
 88        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 89        /// <remarks>In-place conversion is prohibited, please use <see cref="ToLowerInPlace(Span{char}, out int)"/> for
 90        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 91        public static OperationStatus ToLower(ReadOnlySpan<char> source, Span<char> destination, out int charsWritten)
 192            => ChangeCase<ushort, ushort, ToLowerConversion>(MemoryMarshal.Cast<char, ushort>(source), MemoryMarshal.Cas
 93
 94        /// <summary>
 95        /// Copies text from a source buffer to a destination buffer, converting
 96        /// ASCII letters to lowercase during the copy.
 97        /// </summary>
 98        /// <param name="source">The source buffer from which ASCII text is read.</param>
 99        /// <param name="destination">The destination buffer to which lowercase text is written.</param>
 100        /// <param name="charsWritten">The number of characters actually written to <paramref name="destination"/>. It's
 101        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 102        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 103        public static OperationStatus ToLower(ReadOnlySpan<byte> source, Span<char> destination, out int charsWritten)
 0104            => ChangeCase<byte, ushort, ToLowerConversion>(source, MemoryMarshal.Cast<char, ushort>(destination), out ch
 105
 106        /// <summary>
 107        /// Copies text from a source buffer to a destination buffer, converting
 108        /// ASCII letters to lowercase during the copy.
 109        /// </summary>
 110        /// <param name="source">The source buffer from which ASCII text is read.</param>
 111        /// <param name="destination">The destination buffer to which lowercase text is written.</param>
 112        /// <param name="bytesWritten">The number of bytes actually written to <paramref name="destination"/>. It's the 
 113        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 114        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 115        public static OperationStatus ToLower(ReadOnlySpan<char> source, Span<byte> destination, out int bytesWritten)
 0116            => ChangeCase<ushort, byte, ToLowerConversion>(MemoryMarshal.Cast<char, ushort>(source), destination, out by
 117
 118        /// <summary>
 119        /// Performs in-place uppercase conversion.
 120        /// </summary>
 121        /// <param name="value">The ASCII text buffer.</param>
 122        /// <param name="bytesWritten">The number of processed bytes.</param>
 123        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 124        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 125        public static OperationStatus ToLowerInPlace(Span<byte> value, out int bytesWritten)
 0126            => ChangeCase<byte, ToLowerConversion>(value, out bytesWritten);
 127
 128        /// <summary>
 129        /// Performs in-place uppercase conversion.
 130        /// </summary>
 131        /// <param name="value">The ASCII text buffer.</param>
 132        /// <param name="charsWritten">The number of processed characters.</param>
 133        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 134        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 135        public static OperationStatus ToLowerInPlace(Span<char> value, out int charsWritten)
 0136            => ChangeCase<ushort, ToLowerConversion>(MemoryMarshal.Cast<char, ushort>(value), out charsWritten);
 137
 138        /// <summary>
 139        /// Performs in-place lowercase conversion.
 140        /// </summary>
 141        /// <param name="value">The ASCII text buffer.</param>
 142        /// <param name="bytesWritten">The number of processed bytes.</param>
 143        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 144        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 145        public static OperationStatus ToUpperInPlace(Span<byte> value, out int bytesWritten)
 0146            => ChangeCase<byte, ToUpperConversion>(value, out bytesWritten);
 147
 148        /// <summary>
 149        /// Performs in-place lowercase conversion.
 150        /// </summary>
 151        /// <param name="value">The ASCII text buffer.</param>
 152        /// <param name="charsWritten">The number of processed characters.</param>
 153        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 154        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 155        public static OperationStatus ToUpperInPlace(Span<char> value, out int charsWritten)
 0156            => ChangeCase<ushort, ToUpperConversion>(MemoryMarshal.Cast<char, ushort>(value), out charsWritten);
 157
 158        private static unsafe OperationStatus ChangeCase<TFrom, TTo, TCasing>(ReadOnlySpan<TFrom> source, Span<TTo> dest
 159            where TFrom : unmanaged, IBinaryInteger<TFrom>
 160            where TTo : unmanaged, IBinaryInteger<TTo>
 161            where TCasing : struct
 162        {
 1163            if (MemoryMarshal.AsBytes(source).Overlaps(MemoryMarshal.AsBytes(destination)))
 164            {
 0165                ThrowHelper.ThrowInvalidOperationException(ExceptionResource.InvalidOperation_SpanOverlappedOperation);
 166            }
 167
 168            nuint numElementsToConvert;
 169            OperationStatus statusToReturnOnSuccess;
 170
 1171            if (source.Length <= destination.Length)
 172            {
 1173                numElementsToConvert = (uint)source.Length;
 1174                statusToReturnOnSuccess = OperationStatus.Done;
 175            }
 176            else
 177            {
 0178                numElementsToConvert = (uint)destination.Length;
 0179                statusToReturnOnSuccess = OperationStatus.DestinationTooSmall;
 180            }
 181
 1182            fixed (TFrom* pSource = &MemoryMarshal.GetReference(source))
 1183            fixed (TTo* pDestination = &MemoryMarshal.GetReference(destination))
 184            {
 1185                nuint numElementsActuallyConverted = ChangeCase<TFrom, TTo, TCasing>(pSource, pDestination, numElementsT
 1186                Debug.Assert(numElementsActuallyConverted <= numElementsToConvert);
 187
 1188                destinationElementsWritten = (int)numElementsActuallyConverted;
 1189                return (numElementsToConvert == numElementsActuallyConverted) ? statusToReturnOnSuccess : OperationStatu
 190            }
 191        }
 192
 193        private static unsafe OperationStatus ChangeCase<T, TCasing>(Span<T> buffer, out int elementsWritten)
 194            where T : unmanaged, IBinaryInteger<T>
 195            where TCasing : struct
 0196        {
 0197            fixed (T* pBuffer = &MemoryMarshal.GetReference(buffer))
 198            {
 0199                nuint numElementsActuallyConverted = ChangeCase<T, T, TCasing>(pBuffer, pBuffer, (nuint)buffer.Length);
 0200                Debug.Assert(numElementsActuallyConverted <= (nuint)buffer.Length);
 201
 0202                elementsWritten = (int)numElementsActuallyConverted;
 0203                return elementsWritten == buffer.Length ? OperationStatus.Done : OperationStatus.InvalidData;
 204            }
 205        }
 206
 207        private static unsafe nuint ChangeCase<TFrom, TTo, TCasing>(TFrom* pSrc, TTo* pDest, nuint elementCount)
 208            where TFrom : unmanaged, IBinaryInteger<TFrom>
 209            where TTo : unmanaged, IBinaryInteger<TTo>
 210            where TCasing : struct
 211        {
 212            Debug.Assert(typeof(TFrom) == typeof(byte) || typeof(TFrom) == typeof(ushort));
 1213            Debug.Assert(typeof(TTo) == typeof(byte) || typeof(TTo) == typeof(ushort));
 1214            Debug.Assert(typeof(TCasing) == typeof(ToUpperConversion) || typeof(TCasing) == typeof(ToLowerConversion));
 215
 1216            bool sourceIsAscii = (sizeof(TFrom) == 1); // JIT turns this into a const
 1217            bool destIsAscii = (sizeof(TTo) == 1); // JIT turns this into a const
 1218            bool conversionIsWidening = sourceIsAscii && !destIsAscii; // JIT turns this into a const
 219            bool conversionIsNarrowing = !sourceIsAscii && destIsAscii; // JIT turns this into a const
 1220            bool conversionIsWidthPreserving = typeof(TFrom) == typeof(TTo); // JIT turns this into a const
 1221            bool conversionIsToUpper = (typeof(TCasing) == typeof(ToUpperConversion)); // JIT turns this into a const
 1222            uint numInputElementsToConsumeEachVectorizedLoopIteration = (uint)(sizeof(Vector128<byte>) / sizeof(TFrom));
 223
 1224            nuint i = 0;
 225
 226            // The only situation we can't easily optimize is non-hardware-accelerated
 227            // widening or narrowing. In this case, fall back to a naive element-by-element
 228            // loop.
 229
 1230            if (!conversionIsWidthPreserving && !Vector128.IsHardwareAccelerated)
 231            {
 232                goto DrainRemaining;
 233            }
 234
 235            // Process the input as a series of 128-bit blocks.
 236
 1237            if (Vector128.IsHardwareAccelerated && elementCount >= numInputElementsToConsumeEachVectorizedLoopIteration)
 238            {
 239                // Unaligned read and check for non-ASCII data.
 240
 1241                Vector128<TFrom> srcVector = Vector128.LoadUnsafe(ref *pSrc);
 1242                if (VectorContainsNonAsciiChar(srcVector))
 243                {
 244                    goto Drain64;
 245                }
 246
 247                // Now find matching characters and perform case conversion.
 248                // Basically, the (A <= value && value <= Z) check is converted to:
 249                // (value - CONST) <= (Z - A), but using signed instead of unsigned arithmetic.
 250
 1251                TFrom SourceSignedMinValue = TFrom.CreateTruncating(1 << (8 * sizeof(TFrom) - 1));
 1252                Vector128<TFrom> subtractionVector = Vector128.Create(conversionIsToUpper ? (SourceSignedMinValue + TFro
 1253                Vector128<TFrom> comparisonVector = Vector128.Create(SourceSignedMinValue + TFrom.CreateTruncating(26 /*
 1254                Vector128<TFrom> caseConversionVector = Vector128.Create(TFrom.CreateTruncating(0x20)); // works both di
 255
 1256                Vector128<TFrom> matches = SignedLessThan((srcVector - subtractionVector), comparisonVector);
 1257                srcVector ^= (matches & caseConversionVector);
 258
 259                // Now write to the destination.
 260
 1261                ChangeWidthAndWriteTo(srcVector, pDest, 0);
 262
 263                // Now that the first conversion is out of the way, calculate how
 264                // many elements we should skip in order to have future writes be
 265                // aligned.
 266
 1267                uint expectedWriteAlignment = numInputElementsToConsumeEachVectorizedLoopIteration * (uint)sizeof(TTo); 
 1268                i = numInputElementsToConsumeEachVectorizedLoopIteration - ((uint)pDest % expectedWriteAlignment) / (uin
 1269                Debug.Assert((nuint)(&pDest[i]) % expectedWriteAlignment == 0, "Destination buffer wasn't properly align
 270
 271                // Future iterations of this loop will be aligned,
 272                // except for the last iteration.
 273
 1274                while (true)
 275                {
 2276                    Debug.Assert(i <= elementCount, "We overran a buffer somewhere.");
 277
 2278                    if ((elementCount - i) < numInputElementsToConsumeEachVectorizedLoopIteration)
 279                    {
 280                        // If we're about to enter the final iteration of the loop, back up so that
 281                        // we can read one unaligned block. If we've already consumed all the data,
 282                        // jump straight to the end.
 283
 2284                        if (i == elementCount)
 285                        {
 286                            goto Return;
 287                        }
 288
 1289                        i = elementCount - numInputElementsToConsumeEachVectorizedLoopIteration;
 290                    }
 291
 292                    // Unaligned read & check for non-ASCII data.
 293
 1294                    srcVector = Vector128.LoadUnsafe(ref *pSrc, i);
 1295                    if (VectorContainsNonAsciiChar(srcVector))
 296                    {
 297                        goto Drain64;
 298                    }
 299
 300                    // Now find matching characters and perform case conversion.
 301
 1302                    matches = SignedLessThan((srcVector - subtractionVector), comparisonVector);
 1303                    srcVector ^= (matches & caseConversionVector);
 304
 305                    // Now write to the destination.
 306                    // We expect this write to be aligned except for the last run through the loop.
 307
 1308                    ChangeWidthAndWriteTo(srcVector, pDest, i);
 1309                    i += numInputElementsToConsumeEachVectorizedLoopIteration;
 310                }
 311            }
 312
 313        Drain64:
 314
 315            // Attempt to process blocks of 64 input bits.
 316
 317            if (IntPtr.Size >= 8 && (elementCount - i) >= (nuint)(8 / sizeof(TFrom)))
 318            {
 0319                ulong nextBlockAsUInt64 = Unsafe.ReadUnaligned<ulong>(&pSrc[i]);
 0320                if (sourceIsAscii)
 321                {
 0322                    if (!Utf8Utility.AllBytesInUInt64AreAscii(nextBlockAsUInt64))
 323                    {
 324                        goto Drain32;
 325                    }
 0326                    nextBlockAsUInt64 = (conversionIsToUpper)
 0327                        ? Utf8Utility.ConvertAllAsciiBytesInUInt64ToUppercase(nextBlockAsUInt64)
 0328                        : Utf8Utility.ConvertAllAsciiBytesInUInt64ToLowercase(nextBlockAsUInt64);
 329                }
 330                else
 331                {
 0332                    if (!Utf16Utility.AllCharsInUInt64AreAscii(nextBlockAsUInt64))
 333                    {
 334                        goto Drain32;
 335                    }
 0336                    nextBlockAsUInt64 = (conversionIsToUpper)
 0337                        ? Utf16Utility.ConvertAllAsciiCharsInUInt64ToUppercase(nextBlockAsUInt64)
 0338                        : Utf16Utility.ConvertAllAsciiCharsInUInt64ToLowercase(nextBlockAsUInt64);
 339                }
 340
 0341                if (conversionIsWidthPreserving)
 342                {
 0343                    Unsafe.WriteUnaligned(&pDest[i], nextBlockAsUInt64);
 344                }
 345                else
 346                {
 0347                    Debug.Assert(Vector128.IsHardwareAccelerated);
 348
 0349                    Vector128<ulong> blockAsVectorOfUInt64 = Vector128.CreateScalarUnsafe(nextBlockAsUInt64);
 0350                    if (conversionIsWidening)
 351                    {
 0352                        Vector128.StoreUnsafe(Vector128.WidenLower(blockAsVectorOfUInt64.AsByte()), ref *(ushort*)pDest,
 353                    }
 354                    else
 355                    {
 0356                        Vector128<ushort> blockAsVectorOfUInt16 = blockAsVectorOfUInt64.AsUInt16();
 0357                        Vector128<uint> narrowedBlock = Vector128.Narrow(blockAsVectorOfUInt16, blockAsVectorOfUInt16).A
 0358                        Unsafe.WriteUnaligned(&pDest[i], narrowedBlock.ToScalar());
 359                    }
 360                }
 361
 0362                i += (nuint)(8 / sizeof(TFrom));
 363
 364                // If vectorization is not accelerated, turn this into a while loop.
 365
 0366                if (!Vector128.IsHardwareAccelerated)
 367                {
 368                    goto Drain64;
 369                }
 370            }
 371
 372        Drain32:
 373
 374            // Attempt to process blocks of 32 input bits.
 375
 0376            if ((elementCount - i) >= (nuint)(4 / sizeof(TFrom)))
 377            {
 0378                uint nextBlockAsUInt32 = Unsafe.ReadUnaligned<uint>(&pSrc[i]);
 0379                if (sourceIsAscii)
 380                {
 0381                    if (!Utf8Utility.AllBytesInUInt32AreAscii(nextBlockAsUInt32))
 382                    {
 383                        goto DrainRemaining;
 384                    }
 0385                    nextBlockAsUInt32 = (conversionIsToUpper)
 0386                        ? Utf8Utility.ConvertAllAsciiBytesInUInt32ToUppercase(nextBlockAsUInt32)
 0387                        : Utf8Utility.ConvertAllAsciiBytesInUInt32ToLowercase(nextBlockAsUInt32);
 388                }
 389                else
 390                {
 0391                    if (!Utf16Utility.AllCharsInUInt32AreAscii(nextBlockAsUInt32))
 392                    {
 393                        goto DrainRemaining;
 394                    }
 0395                    nextBlockAsUInt32 = (conversionIsToUpper)
 0396                        ? Utf16Utility.ConvertAllAsciiCharsInUInt32ToUppercase(nextBlockAsUInt32)
 0397                        : Utf16Utility.ConvertAllAsciiCharsInUInt32ToLowercase(nextBlockAsUInt32);
 398                }
 399
 0400                if (conversionIsWidthPreserving)
 401                {
 0402                    Unsafe.WriteUnaligned(&pDest[i], nextBlockAsUInt32);
 403                }
 404                else
 405                {
 0406                    Debug.Assert(Vector128.IsHardwareAccelerated);
 407
 0408                    Vector128<uint> blockAsVectorOfUInt32 = Vector128.CreateScalarUnsafe(nextBlockAsUInt32);
 0409                    if (conversionIsWidening)
 410                    {
 0411                        Vector128<ulong> widenedBlock = Vector128.WidenLower(blockAsVectorOfUInt32.AsByte()).AsUInt64();
 0412                        Unsafe.WriteUnaligned(&pDest[i], widenedBlock.ToScalar());
 413                    }
 414                    else
 415                    {
 0416                        Vector128<ushort> blockAsVectorOfUInt16 = blockAsVectorOfUInt32.AsUInt16();
 0417                        Vector128<ushort> narrowedBlock = Vector128.Narrow(blockAsVectorOfUInt16, blockAsVectorOfUInt16)
 0418                        Unsafe.WriteUnaligned(&pDest[i], narrowedBlock.ToScalar());
 419                    }
 420                }
 421
 0422                i += (nuint)(4 / sizeof(TFrom));
 423
 424                // If vectorization is not accelerated or we're on 32-bit,
 425                // turn this into a while loop.
 426
 427                if (IntPtr.Size < 8 || !Vector128.IsHardwareAccelerated)
 428                {
 0429                    goto Drain32;
 430                }
 431            }
 432
 433        DrainRemaining:
 434
 435            // Process single elements at a time.
 436
 0437            for (; i < elementCount; i++)
 438            {
 0439                uint element = uint.CreateTruncating(pSrc[i]);
 0440                if (!UnicodeUtility.IsAsciiCodePoint(element))
 441                {
 442                    break;
 443                }
 444
 0445                if (conversionIsToUpper)
 446                {
 0447                    if (UnicodeUtility.IsInRangeInclusive(element, 'a', 'z'))
 448                    {
 0449                        element -= 0x20u; // lowercase to uppercase
 450                    }
 451                }
 452                else
 453                {
 0454                    if (UnicodeUtility.IsInRangeInclusive(element, 'A', 'Z'))
 455                    {
 0456                        element += 0x20u; // uppercase to lowercase
 457                    }
 458                }
 0459                pDest[i] = TTo.CreateTruncating(element);
 460            }
 461
 462        Return:
 463
 1464            return i;
 465        }
 466
 467        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 468        private static unsafe void ChangeWidthAndWriteTo<TFrom, TTo>(Vector128<TFrom> vector, TTo* pDest, nuint elementO
 469            where TFrom : unmanaged
 470            where TTo : unmanaged
 471        {
 2472            if (sizeof(TFrom) == sizeof(TTo))
 473            {
 474                // no width change needed
 2475                Vector128.StoreUnsafe(vector.As<TFrom, TTo>(), ref *pDest, elementOffset);
 476            }
 0477            else if (sizeof(TFrom) == 1 && sizeof(TTo) == 2)
 478            {
 479                // widening operation required
 0480                if (Vector256.IsHardwareAccelerated)
 481                {
 0482                    Vector256<ushort> wide = Vector256.WidenLower(vector.AsByte().ToVector256Unsafe());
 0483                    Vector256.StoreUnsafe(wide, ref *(ushort*)pDest, elementOffset);
 484                }
 485                else
 486                {
 0487                    Vector128.StoreUnsafe(Vector128.WidenLower(vector.AsByte()), ref *(ushort*)pDest, elementOffset);
 0488                    Vector128.StoreUnsafe(Vector128.WidenUpper(vector.AsByte()), ref *(ushort*)pDest, elementOffset + 8)
 489                }
 490            }
 0491            else if (sizeof(TFrom) == 2 && sizeof(TTo) == 1)
 492            {
 493                // narrowing operation required, we know data is all-ASCII so use extract helper
 0494                Vector128<byte> narrow = ExtractAsciiVector(vector.AsUInt16(), vector.AsUInt16());
 0495                narrow.StoreLowerUnsafe(ref *(byte*)pDest, elementOffset);
 496            }
 497            else
 498            {
 0499                Debug.Fail("Unknown types.");
 500                throw new NotSupportedException();
 501            }
 502        }
 503
 504        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 505        private static unsafe Vector128<T> SignedLessThan<T>(Vector128<T> left, Vector128<T> right)
 506            where T : unmanaged
 507        {
 2508            if (sizeof(T) == 1)
 509            {
 0510                return Vector128.LessThan(left.AsSByte(), right.AsSByte()).As<sbyte, T>();
 511            }
 2512            else if (sizeof(T) == 2)
 513            {
 2514                return Vector128.LessThan(left.AsInt16(), right.AsInt16()).As<short, T>();
 515            }
 516            else
 517            {
 0518                throw new NotSupportedException();
 519            }
 520        }
 521
 522        private struct ToUpperConversion { }
 523        private struct ToLowerConversion { }
 524    }
 525}
 526

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Diagnostics;
 5using System.Runtime.CompilerServices;
 6using System.Runtime.InteropServices;
 7using System.Runtime.Intrinsics;
 8using System.Runtime.Intrinsics.X86;
 9
 10namespace System.Text
 11{
 12    public static partial class Ascii
 13    {
 14        /// <summary>
 15        /// Determines whether the provided value contains only ASCII bytes.
 16        /// </summary>
 17        /// <param name="value">The value to inspect.</param>
 18        /// <returns>True if <paramref name="value"/> contains only ASCII bytes or is
 19        /// empty; False otherwise.</returns>
 20        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 21        public static bool IsValid(ReadOnlySpan<byte> value) =>
 022            IsValidCore(ref MemoryMarshal.GetReference(value), value.Length);
 23
 24        /// <summary>
 25        /// Determines whether the provided value contains only ASCII chars.
 26        /// </summary>
 27        /// <param name="value">The value to inspect.</param>
 28        /// <returns>True if <paramref name="value"/> contains only ASCII chars or is
 29        /// empty; False otherwise.</returns>
 30        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 31        public static bool IsValid(ReadOnlySpan<char> value) =>
 132            IsValidCore(ref Unsafe.As<char, ushort>(ref MemoryMarshal.GetReference(value)), value.Length);
 33
 34        /// <summary>
 35        /// Determines whether the provided value is ASCII byte.
 36        /// </summary>
 37        /// <param name="value">The value to inspect.</param>
 38        /// <returns>True if <paramref name="value"/> is ASCII, False otherwise.</returns>
 039        public static bool IsValid(byte value) => value <= 127;
 40
 41        /// <summary>
 42        /// Determines whether the provided value is ASCII char.
 43        /// </summary>
 44        /// <param name="value">The value to inspect.</param>
 45        /// <returns>True if <paramref name="value"/> is ASCII, False otherwise.</returns>
 046        public static bool IsValid(char value) => value <= 127;
 47
 48        private static unsafe bool IsValidCore<T>(ref T searchSpace, int length) where T : unmanaged
 49        {
 150            Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));
 51
 152            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
 53            {
 154                uint elementsPerUlong = (uint)(sizeof(ulong) / sizeof(T));
 55
 156                if (length < elementsPerUlong)
 57                {
 058                    if (typeof(T) == typeof(byte) && length >= sizeof(uint))
 59                    {
 60                        // Process byte inputs with lengths [4, 7]
 061                        return AllBytesInUInt32AreAscii(
 062                            Unsafe.ReadUnaligned<uint>(ref Unsafe.As<T, byte>(ref searchSpace)) |
 063                            Unsafe.ReadUnaligned<uint>(ref Unsafe.As<T, byte>(ref Unsafe.Add(ref searchSpace, length - s
 64                    }
 65
 66                    // Process inputs with lengths [0, 3]
 067                    for (nuint j = 0; j < (uint)length; j++)
 68                    {
 069                        if (typeof(T) == typeof(byte)
 070                            ? (Unsafe.BitCast<T, byte>(Unsafe.Add(ref searchSpace, j)) > 127)
 071                            : (Unsafe.BitCast<T, char>(Unsafe.Add(ref searchSpace, j)) > 127))
 72                        {
 073                            return false;
 74                        }
 75                    }
 76
 077                    return true;
 78                }
 79
 180                nuint i = 0;
 81
 82                // If vectorization isn't supported, process 16 bytes at a time.
 183                if (!Vector128.IsHardwareAccelerated && length > 2 * elementsPerUlong)
 84                {
 085                    nuint finalStart = (nuint)length - 2 * elementsPerUlong;
 86
 087                    for (; i < finalStart; i += 2 * elementsPerUlong)
 88                    {
 089                        if (!AllCharsInUInt64AreAscii<T>(
 090                            Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<T, byte>(ref Unsafe.Add(ref searchSpace, i))) |
 091                            Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<T, byte>(ref Unsafe.Add(ref searchSpace, i + eleme
 92                        {
 093                            return false;
 94                        }
 95                    }
 96
 097                    i = finalStart;
 98                }
 99
 100                // Process the last [8, 16] bytes.
 1101                return AllCharsInUInt64AreAscii<T>(
 1102                    Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<T, byte>(ref Unsafe.Add(ref searchSpace, i))) |
 1103                    Unsafe.ReadUnaligned<ulong>(ref Unsafe.Subtract(ref Unsafe.As<T, byte>(ref Unsafe.Add(ref searchSpac
 104            }
 105
 0106            ref T searchSpaceEnd = ref Unsafe.Add(ref searchSpace, length);
 107
 108            // Process inputs with lengths [16, 32] bytes.
 0109            if (length <= 2 * Vector128<T>.Count)
 110            {
 0111                return AllCharsInVectorAreAscii(
 0112                    Vector128.LoadUnsafe(ref searchSpace) |
 0113                    Vector128.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, Vector128<T>.Count)));
 114            }
 115
 0116            if (Avx.IsSupported)
 117            {
 118                // Process inputs with lengths [33, 64] bytes.
 0119                if (length <= 2 * Vector256<T>.Count)
 120                {
 0121                    return AllCharsInVectorAreAscii(
 0122                        Vector256.LoadUnsafe(ref searchSpace) |
 0123                        Vector256.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, Vector256<T>.Count)));
 124                }
 125
 126                // Process long inputs 128 bytes at a time.
 0127                if (length > 4 * Vector256<T>.Count)
 128                {
 129                    // Process the first 128 bytes.
 0130                    if (!AllCharsInVectorAreAscii(
 0131                        Vector256.LoadUnsafe(ref searchSpace) |
 0132                        Vector256.LoadUnsafe(ref searchSpace, (nuint)Vector256<T>.Count) |
 0133                        Vector256.LoadUnsafe(ref searchSpace, 2 * (nuint)Vector256<T>.Count) |
 0134                        Vector256.LoadUnsafe(ref searchSpace, 3 * (nuint)Vector256<T>.Count)))
 135                    {
 0136                        return false;
 137                    }
 138
 0139                    nuint i = 4 * (nuint)Vector256<T>.Count;
 140
 141                    // Try to opportunistically align the reads below. The input isn't pinned, so the GC
 142                    // is free to move the references. We're therefore assuming that reads may still be unaligned.
 143                    // They may also be unaligned if the input chars aren't 2-byte aligned.
 0144                    nuint misalignedElements = Unsafe.OpportunisticMisalignment(ref searchSpace, (uint)Vector256<byte>.C
 0145                    i -= misalignedElements;
 0146                    Debug.Assert((int)i > 3 * Vector256<T>.Count);
 147
 0148                    nuint finalStart = (nuint)length - 4 * (nuint)Vector256<T>.Count;
 149
 0150                    for (; i < finalStart; i += 4 * (nuint)Vector256<T>.Count)
 151                    {
 0152                        ref T current = ref Unsafe.Add(ref searchSpace, i);
 153
 0154                        if (!AllCharsInVectorAreAscii(
 0155                            Vector256.LoadUnsafe(ref current) |
 0156                            Vector256.LoadUnsafe(ref current, (nuint)Vector256<T>.Count) |
 0157                            Vector256.LoadUnsafe(ref current, 2 * (nuint)Vector256<T>.Count) |
 0158                            Vector256.LoadUnsafe(ref current, 3 * (nuint)Vector256<T>.Count)))
 159                        {
 0160                            return false;
 161                        }
 162                    }
 163
 0164                    searchSpace = ref Unsafe.Add(ref searchSpace, finalStart);
 165                }
 166
 167                // Process the last [1, 128] bytes.
 168                // The search space has at least 2 * Vector256 bytes available to read.
 169                // We process the first 2 and last 2 vectors, which may overlap.
 0170                return AllCharsInVectorAreAscii(
 0171                    Vector256.LoadUnsafe(ref searchSpace) |
 0172                    Vector256.LoadUnsafe(ref searchSpace, (nuint)Vector256<T>.Count) |
 0173                    Vector256.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, 2 * Vector256<T>.Count)) |
 0174                    Vector256.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, Vector256<T>.Count)));
 175            }
 176            else
 177            {
 178                // Process long inputs 64 bytes at a time.
 0179                if (length > 4 * Vector128<T>.Count)
 180                {
 181                    // Process the first 64 bytes.
 0182                    if (!AllCharsInVectorAreAscii(
 0183                        Vector128.LoadUnsafe(ref searchSpace) |
 0184                        Vector128.LoadUnsafe(ref searchSpace, (nuint)Vector128<T>.Count) |
 0185                        Vector128.LoadUnsafe(ref searchSpace, 2 * (nuint)Vector128<T>.Count) |
 0186                        Vector128.LoadUnsafe(ref searchSpace, 3 * (nuint)Vector128<T>.Count)))
 187                    {
 0188                        return false;
 189                    }
 190
 0191                    nuint i = 4 * (nuint)Vector128<T>.Count;
 192
 193                    // Try to opportunistically align the reads below. The input isn't pinned, so the GC
 194                    // is free to move the references. We're therefore assuming that reads may still be unaligned.
 195                    // They may also be unaligned if the input chars aren't 2-byte aligned.
 0196                    nuint misalignedElements = Unsafe.OpportunisticMisalignment(ref searchSpace, (uint)Vector128<byte>.C
 0197                    i -= misalignedElements;
 0198                    Debug.Assert((int)i > 3 * Vector128<T>.Count);
 199
 0200                    nuint finalStart = (nuint)length - 4 * (nuint)Vector128<T>.Count;
 201
 0202                    for (; i < finalStart; i += 4 * (nuint)Vector128<T>.Count)
 203                    {
 0204                        ref T current = ref Unsafe.Add(ref searchSpace, i);
 205
 0206                        if (!AllCharsInVectorAreAscii(
 0207                            Vector128.LoadUnsafe(ref current) |
 0208                            Vector128.LoadUnsafe(ref current, (nuint)Vector128<T>.Count) |
 0209                            Vector128.LoadUnsafe(ref current, 2 * (nuint)Vector128<T>.Count) |
 0210                            Vector128.LoadUnsafe(ref current, 3 * (nuint)Vector128<T>.Count)))
 211                        {
 0212                            return false;
 213                        }
 214                    }
 215
 0216                    searchSpace = ref Unsafe.Add(ref searchSpace, finalStart);
 217                }
 218
 219                // Process the last [1, 64] bytes.
 220                // The search space has at least 2 * Vector128 bytes available to read.
 221                // We process the first 2 and last 2 vectors, which may overlap.
 0222                return AllCharsInVectorAreAscii(
 0223                    Vector128.LoadUnsafe(ref searchSpace) |
 0224                    Vector128.LoadUnsafe(ref searchSpace, (nuint)Vector128<T>.Count) |
 0225                    Vector128.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, 2 * Vector128<T>.Count)) |
 0226                    Vector128.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, Vector128<T>.Count)));
 227            }
 228        }
 229    }
 230}
 231

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Diagnostics;
 5using System.Numerics;
 6using System.Runtime.CompilerServices;
 7using System.Runtime.InteropServices;
 8using System.Runtime.Intrinsics;
 9using System.Runtime.Intrinsics.Arm;
 10using System.Runtime.Intrinsics.Wasm;
 11using System.Runtime.Intrinsics.X86;
 12
 13namespace System.Text
 14{
 15    public static partial class Ascii
 16    {
 17        /// <summary>
 18        /// Determines whether the provided buffers contain equal ASCII characters.
 19        /// </summary>
 20        /// <param name="left">The buffer to compare with <paramref name="right" />.</param>
 21        /// <param name="right">The buffer to compare with <paramref name="left" />.</param>
 22        /// <returns><see langword="true" /> if the corresponding elements in <paramref name="left" /> and <paramref nam
 23        /// <remarks>If both buffers contain equal, but non-ASCII characters, the method returns <see langword="false" /
 24        public static bool Equals(ReadOnlySpan<byte> left, ReadOnlySpan<byte> right)
 025            => left.Length == right.Length
 026            && Equals<byte, byte, PlainLoader<byte>>(ref MemoryMarshal.GetReference(left), ref MemoryMarshal.GetReferenc
 27
 28        /// <inheritdoc cref="Equals(ReadOnlySpan{byte}, ReadOnlySpan{byte})"/>
 29        public static bool Equals(ReadOnlySpan<byte> left, ReadOnlySpan<char> right)
 030            => left.Length == right.Length
 031            && Equals<byte, ushort, WideningLoader>(ref MemoryMarshal.GetReference(left), ref Unsafe.As<char, ushort>(re
 32
 33        /// <inheritdoc cref="Equals(ReadOnlySpan{byte}, ReadOnlySpan{char})"/>
 34        public static bool Equals(ReadOnlySpan<char> left, ReadOnlySpan<byte> right)
 035            => Equals(right, left);
 36
 37        /// <inheritdoc cref="Equals(ReadOnlySpan{byte}, ReadOnlySpan{char})"/>
 38        public static bool Equals(ReadOnlySpan<char> left, ReadOnlySpan<char> right)
 039            => left.Length == right.Length
 040            && Equals<ushort, ushort, PlainLoader<ushort>>(ref Unsafe.As<char, ushort>(ref MemoryMarshal.GetReference(le
 41
 42        private static bool Equals<TLeft, TRight, TLoader>(ref TLeft left, ref TRight right, nuint length)
 43            where TLeft : unmanaged, INumberBase<TLeft>
 44            where TRight : unmanaged, INumberBase<TRight>
 45            where TLoader : struct, ILoader<TLeft, TRight>
 46        {
 047            Debug.Assert(
 048                (typeof(TLeft) == typeof(byte) && typeof(TRight) == typeof(byte))
 049             || (typeof(TLeft) == typeof(byte) && typeof(TRight) == typeof(ushort))
 050             || (typeof(TLeft) == typeof(ushort) && typeof(TRight) == typeof(ushort)));
 51
 052            if (!Vector128.IsHardwareAccelerated || length < (uint)Vector128<TRight>.Count)
 53            {
 054                for (nuint i = 0; i < length; ++i)
 55                {
 056                    uint valueA = uint.CreateTruncating(Unsafe.Add(ref left, i));
 057                    uint valueB = uint.CreateTruncating(Unsafe.Add(ref right, i));
 58
 059                    if (valueA != valueB || !UnicodeUtility.IsAsciiCodePoint(valueA))
 60                    {
 061                        return false;
 62                    }
 63                }
 64            }
 065            else if (Vector512.IsHardwareAccelerated && length >= (uint)Vector512<TLeft>.Count)
 66            {
 067                ref TLeft currentLeftSearchSpace = ref left;
 068                ref TRight currentRightSearchSpace = ref right;
 69                // Add Vector512<TLeft>.Count because TLeft == TRight
 70                // Or we are in the Widen case where we iterate 2 * TRight.Count which is the same as TLeft.Count
 071                Debug.Assert(Vector512<TLeft>.Count == Vector512<TRight>.Count
 072                    || (typeof(TLoader) == typeof(WideningLoader) && Vector512<TLeft>.Count == Vector512<TRight>.Count *
 073                ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector
 74
 75                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
 76                do
 77                {
 078                    if (!TLoader.EqualAndAscii512(ref currentLeftSearchSpace, ref currentRightSearchSpace))
 79                    {
 080                        return false;
 81                    }
 82
 083                    currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, Vector512<TLeft>.Count);
 084                    currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, Vector512<TLeft>.Count);
 85                }
 086                while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd));
 87
 88                // If any elements remain, process the last vector in the search space.
 089                if (length % (uint)Vector512<TLeft>.Count != 0)
 90                {
 091                    ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref left, length - (uint)Vector512<TLeft>.Count)
 092                    return TLoader.EqualAndAscii512(ref oneVectorAwayFromLeftEnd, ref oneVectorAwayFromRightEnd);
 93                }
 94            }
 095            else if (Avx.IsSupported && length >= (uint)Vector256<TLeft>.Count)
 96            {
 097                ref TLeft currentLeftSearchSpace = ref left;
 098                ref TRight currentRightSearchSpace = ref right;
 99                // Add Vector256<TLeft>.Count because TLeft == TRight
 100                // Or we are in the Widen case where we iterate 2 * TRight.Count which is the same as TLeft.Count
 0101                Debug.Assert(Vector256<TLeft>.Count == Vector256<TRight>.Count
 0102                    || (typeof(TLoader) == typeof(WideningLoader) && Vector256<TLeft>.Count == Vector256<TRight>.Count *
 0103                ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector
 104
 105                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
 106                do
 107                {
 0108                    if (!TLoader.EqualAndAscii256(ref currentLeftSearchSpace, ref currentRightSearchSpace))
 109                    {
 0110                        return false;
 111                    }
 112
 0113                    currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, Vector256<TLeft>.Count);
 0114                    currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, Vector256<TLeft>.Count);
 115                }
 0116                while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd));
 117
 118                // If any elements remain, process the last vector in the search space.
 0119                if (length % (uint)Vector256<TLeft>.Count != 0)
 120                {
 0121                    ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref left, length - (uint)Vector256<TLeft>.Count)
 0122                    return TLoader.EqualAndAscii256(ref oneVectorAwayFromLeftEnd, ref oneVectorAwayFromRightEnd);
 123                }
 124            }
 125            else
 126            {
 0127                ref TLeft currentLeftSearchSpace = ref left;
 0128                ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref currentLeftSearchSpace, length - (uint)Vector128
 0129                ref TRight currentRightSearchSpace = ref right;
 0130                ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector
 131
 132                Vector128<TRight> leftValues;
 133                Vector128<TRight> rightValues;
 134
 135                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
 136                do
 137                {
 138                    // it's OK to widen the bytes, it's NOT OK to narrow the chars (we could lose some information)
 0139                    leftValues = TLoader.Load128(ref currentLeftSearchSpace);
 0140                    rightValues = Vector128.LoadUnsafe(ref currentRightSearchSpace);
 141
 0142                    if (leftValues != rightValues || !AllCharsInVectorAreAscii(leftValues))
 143                    {
 0144                        return false;
 145                    }
 146
 0147                    currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector128<TRight>.Count)
 0148                    currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, (uint)Vector128<TRight>.Count);
 149                }
 0150                while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd));
 151
 152                // If any elements remain, process the last vector in the search space.
 0153                if (length % (uint)Vector128<TRight>.Count != 0)
 154                {
 0155                    leftValues = TLoader.Load128(ref oneVectorAwayFromLeftEnd);
 0156                    rightValues = Vector128.LoadUnsafe(ref oneVectorAwayFromRightEnd);
 157
 0158                    if (leftValues != rightValues || !AllCharsInVectorAreAscii(leftValues))
 159                    {
 0160                        return false;
 161                    }
 162                }
 163            }
 164
 0165            return true;
 166        }
 167
 168        /// <summary>
 169        /// Determines whether the provided buffers contain equal ASCII characters, ignoring case considerations.
 170        /// </summary>
 171        /// <param name="left">The buffer to compare with <paramref name="right" />.</param>
 172        /// <param name="right">The buffer to compare with <paramref name="left" />.</param>
 173        /// <returns><see langword="true" /> if the corresponding elements in <paramref name="left" /> and <paramref nam
 174        /// <remarks>If both buffers contain equal, but non-ASCII characters, the method returns <see langword="false" /
 175        public static bool EqualsIgnoreCase(ReadOnlySpan<byte> left, ReadOnlySpan<byte> right)
 0176            => left.Length == right.Length
 0177            && EqualsIgnoreCase<byte, byte, PlainLoader<byte>>(ref MemoryMarshal.GetReference(left), ref MemoryMarshal.G
 178
 179        /// <inheritdoc cref="EqualsIgnoreCase(ReadOnlySpan{byte}, ReadOnlySpan{byte})"/>
 180        public static bool EqualsIgnoreCase(ReadOnlySpan<byte> left, ReadOnlySpan<char> right)
 0181            => left.Length == right.Length
 0182            && EqualsIgnoreCase<byte, ushort, WideningLoader>(ref MemoryMarshal.GetReference(left), ref Unsafe.As<char, 
 183
 184        /// <inheritdoc cref="EqualsIgnoreCase(ReadOnlySpan{byte}, ReadOnlySpan{byte})"/>
 185        public static bool EqualsIgnoreCase(ReadOnlySpan<char> left, ReadOnlySpan<byte> right)
 0186            => EqualsIgnoreCase(right, left);
 187
 188        /// <inheritdoc cref="EqualsIgnoreCase(ReadOnlySpan{byte}, ReadOnlySpan{byte})"/>
 189        public static bool EqualsIgnoreCase(ReadOnlySpan<char> left, ReadOnlySpan<char> right)
 0190            => left.Length == right.Length
 0191            && EqualsIgnoreCase<ushort, ushort, PlainLoader<ushort>>(ref Unsafe.As<char, ushort>(ref MemoryMarshal.GetRe
 192
 193        internal static bool EqualsIgnoreCase(ref char left, ref char right, nuint length) =>
 0194            EqualsIgnoreCase<ushort, ushort, PlainLoader<ushort>>(ref Unsafe.As<char, ushort>(ref left), ref Unsafe.As<c
 195
 196        private static bool EqualsIgnoreCase<TLeft, TRight, TLoader>(ref TLeft left, ref TRight right, nuint length)
 197            where TLeft : unmanaged, INumberBase<TLeft>
 198            where TRight : unmanaged, INumberBase<TRight>
 199            where TLoader : ILoader<TLeft, TRight>
 200        {
 0201            Debug.Assert(
 0202                (typeof(TLeft) == typeof(byte) && typeof(TRight) == typeof(byte))
 0203             || (typeof(TLeft) == typeof(byte) && typeof(TRight) == typeof(ushort))
 0204             || (typeof(TLeft) == typeof(ushort) && typeof(TRight) == typeof(ushort)));
 205
 0206            if (!Vector128.IsHardwareAccelerated || length < (uint)Vector128<TRight>.Count)
 207            {
 0208                for (nuint i = 0; i < length; ++i)
 209                {
 0210                    uint valueA = uint.CreateTruncating(Unsafe.Add(ref left, i));
 0211                    uint valueB = uint.CreateTruncating(Unsafe.Add(ref right, i));
 212
 0213                    if (!UnicodeUtility.IsAsciiCodePoint(valueA | valueB))
 214                    {
 0215                        return false;
 216                    }
 217
 0218                    if (valueA == valueB)
 219                    {
 220                        continue; // exact match
 221                    }
 222
 0223                    valueA |= 0x20u;
 0224                    if (valueA - 'a' > 'z' - 'a')
 225                    {
 0226                        return false; // not exact match, and first input isn't in [A-Za-z]
 227                    }
 228
 0229                    if (valueA != (valueB | 0x20u))
 230                    {
 0231                        return false;
 232                    }
 233                }
 234            }
 0235            else if (Vector512.IsHardwareAccelerated && length >= (uint)Vector512<TRight>.Count)
 236            {
 0237                ref TLeft currentLeftSearchSpace = ref left;
 0238                ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref currentLeftSearchSpace, length - (uint)Vector512
 0239                ref TRight currentRightSearchSpace = ref right;
 0240                ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector
 241
 242                Vector512<TRight> leftValues;
 243                Vector512<TRight> rightValues;
 244
 0245                Vector512<TRight> loweringMask = Vector512.Create(TRight.CreateTruncating(0x20));
 0246                Vector512<TRight> vecA = Vector512.Create(TRight.CreateTruncating('a'));
 0247                Vector512<TRight> vecZMinusA = Vector512.Create(TRight.CreateTruncating(('z' - 'a')));
 248
 249                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
 250                do
 251                {
 0252                    leftValues = TLoader.Load512(ref currentLeftSearchSpace);
 0253                    rightValues = Vector512.LoadUnsafe(ref currentRightSearchSpace);
 0254                    if (!AllCharsInVectorAreAscii(leftValues | rightValues))
 255                    {
 0256                        return false;
 257                    }
 258
 0259                    Vector512<TRight> notEquals = ~Vector512.Equals(leftValues, rightValues);
 260
 0261                    if (notEquals != Vector512<TRight>.Zero)
 262                    {
 263                        // not exact match
 264
 0265                        leftValues |= loweringMask;
 0266                        rightValues |= loweringMask;
 267
 0268                        if (Vector512.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != right
 269                        {
 0270                            return false; // first input isn't in [A-Za-z], and not exact match of lowered
 271                        }
 272                    }
 273
 0274                    currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector512<TRight>.Count)
 0275                    currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, (uint)Vector512<TRight>.Count);
 276                }
 0277                while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd));
 278
 279                // If any elements remain, process the last vector in the search space.
 0280                if (length % (uint)Vector512<TRight>.Count != 0)
 281                {
 0282                    leftValues = TLoader.Load512(ref oneVectorAwayFromLeftEnd);
 0283                    rightValues = Vector512.LoadUnsafe(ref oneVectorAwayFromRightEnd);
 284
 0285                    if (!AllCharsInVectorAreAscii(leftValues | rightValues))
 286                    {
 0287                        return false;
 288                    }
 289
 0290                    Vector512<TRight> notEquals = ~Vector512.Equals(leftValues, rightValues);
 291
 0292                    if (notEquals != Vector512<TRight>.Zero)
 293                    {
 294                        // not exact match
 295
 0296                        leftValues |= loweringMask;
 0297                        rightValues |= loweringMask;
 298
 0299                        if (Vector512.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != right
 300                        {
 0301                            return false; // first input isn't in [A-Za-z], and not exact match of lowered
 302                        }
 303                    }
 304                }
 305            }
 0306            else if (Avx.IsSupported && length >= (uint)Vector256<TRight>.Count)
 307            {
 0308                ref TLeft currentLeftSearchSpace = ref left;
 0309                ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref currentLeftSearchSpace, length - (uint)Vector256
 0310                ref TRight currentRightSearchSpace = ref right;
 0311                ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector
 312
 313                Vector256<TRight> leftValues;
 314                Vector256<TRight> rightValues;
 315
 0316                Vector256<TRight> loweringMask = Vector256.Create(TRight.CreateTruncating(0x20));
 0317                Vector256<TRight> vecA = Vector256.Create(TRight.CreateTruncating('a'));
 0318                Vector256<TRight> vecZMinusA = Vector256.Create(TRight.CreateTruncating(('z' - 'a')));
 319
 320                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
 321                do
 322                {
 0323                    leftValues = TLoader.Load256(ref currentLeftSearchSpace);
 0324                    rightValues = Vector256.LoadUnsafe(ref currentRightSearchSpace);
 325
 0326                    if (!AllCharsInVectorAreAscii(leftValues | rightValues))
 327                    {
 0328                        return false;
 329                    }
 330
 0331                    Vector256<TRight> notEquals = ~Vector256.Equals(leftValues, rightValues);
 332
 0333                    if (notEquals != Vector256<TRight>.Zero)
 334                    {
 335                        // not exact match
 336
 0337                        leftValues |= loweringMask;
 0338                        rightValues |= loweringMask;
 339
 0340                        if (Vector256.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != right
 341                        {
 0342                            return false; // first input isn't in [A-Za-z], and not exact match of lowered
 343                        }
 344                    }
 345
 0346                    currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector256<TRight>.Count)
 0347                    currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, (uint)Vector256<TRight>.Count);
 348                }
 0349                while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd));
 350
 351                // If any elements remain, process the last vector in the search space.
 0352                if (length % (uint)Vector256<TRight>.Count != 0)
 353                {
 0354                    leftValues = TLoader.Load256(ref oneVectorAwayFromLeftEnd);
 0355                    rightValues = Vector256.LoadUnsafe(ref oneVectorAwayFromRightEnd);
 356
 0357                    if (!AllCharsInVectorAreAscii(leftValues | rightValues))
 358                    {
 0359                        return false;
 360                    }
 361
 0362                    Vector256<TRight> notEquals = ~Vector256.Equals(leftValues, rightValues);
 363
 0364                    if (notEquals != Vector256<TRight>.Zero)
 365                    {
 366                        // not exact match
 367
 0368                        leftValues |= loweringMask;
 0369                        rightValues |= loweringMask;
 370
 0371                        if (Vector256.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != right
 372                        {
 0373                            return false; // first input isn't in [A-Za-z], and not exact match of lowered
 374                        }
 375                    }
 376                }
 377            }
 378            else
 379            {
 0380                ref TLeft currentLeftSearchSpace = ref left;
 0381                ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref currentLeftSearchSpace, length - (uint)Vector128
 0382                ref TRight currentRightSearchSpace = ref right;
 0383                ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector
 384
 385                Vector128<TRight> leftValues;
 386                Vector128<TRight> rightValues;
 387
 0388                Vector128<TRight> loweringMask = Vector128.Create(TRight.CreateTruncating(0x20));
 0389                Vector128<TRight> vecA = Vector128.Create(TRight.CreateTruncating('a'));
 0390                Vector128<TRight> vecZMinusA = Vector128.Create(TRight.CreateTruncating(('z' - 'a')));
 391
 392                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
 393                do
 394                {
 395                    // it's OK to widen the bytes, it's NOT OK to narrow the chars (we could lose some information)
 0396                    leftValues = TLoader.Load128(ref currentLeftSearchSpace);
 0397                    rightValues = Vector128.LoadUnsafe(ref currentRightSearchSpace);
 398
 0399                    if (!AllCharsInVectorAreAscii(leftValues | rightValues))
 400                    {
 0401                        return false;
 402                    }
 403
 0404                    Vector128<TRight> notEquals = ~Vector128.Equals(leftValues, rightValues);
 405
 0406                    if (notEquals != Vector128<TRight>.Zero)
 407                    {
 408                        // not exact match
 409
 0410                        leftValues |= loweringMask;
 0411                        rightValues |= loweringMask;
 412
 0413                        if (Vector128.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != right
 414                        {
 0415                            return false; // first input isn't in [A-Za-z], and not exact match of lowered
 416                        }
 417                    }
 418
 0419                    currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector128<TRight>.Count)
 0420                    currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, (uint)Vector128<TRight>.Count);
 421                }
 0422                while (Unsafe.IsAddressLessThanOrEqualTo(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd));
 423
 424                // If any elements remain, process the last vector in the search space.
 0425                if (length % (uint)Vector128<TRight>.Count != 0)
 426                {
 0427                    leftValues = TLoader.Load128(ref oneVectorAwayFromLeftEnd);
 0428                    rightValues = Vector128.LoadUnsafe(ref oneVectorAwayFromRightEnd);
 429
 0430                    if (!AllCharsInVectorAreAscii(leftValues | rightValues))
 431                    {
 0432                        return false;
 433                    }
 434
 0435                    Vector128<TRight> notEquals = ~Vector128.Equals(leftValues, rightValues);
 436
 0437                    if (notEquals != Vector128<TRight>.Zero)
 438                    {
 439                        // not exact match
 440
 0441                        leftValues |= loweringMask;
 0442                        rightValues |= loweringMask;
 443
 0444                        if (Vector128.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != right
 445                        {
 0446                            return false; // first input isn't in [A-Za-z], and not exact match of lowered
 447                        }
 448                    }
 449                }
 450            }
 451
 0452            return true;
 453        }
 454
 455        private interface ILoader<TLeft, TRight>
 456            where TLeft : unmanaged, INumberBase<TLeft>
 457            where TRight : unmanaged, INumberBase<TRight>
 458        {
 459            static abstract Vector128<TRight> Load128(ref TLeft ptr);
 460            static abstract Vector256<TRight> Load256(ref TLeft ptr);
 461            static abstract Vector512<TRight> Load512(ref TLeft ptr);
 462            static abstract bool EqualAndAscii256(ref TLeft left, ref TRight right);
 463            static abstract bool EqualAndAscii512(ref TLeft left, ref TRight right);
 464        }
 465
 466        private readonly struct PlainLoader<T> : ILoader<T, T> where T : unmanaged, INumberBase<T>
 467        {
 0468            public static Vector128<T> Load128(ref T ptr) => Vector128.LoadUnsafe(ref ptr);
 0469            public static Vector256<T> Load256(ref T ptr) => Vector256.LoadUnsafe(ref ptr);
 0470            public static Vector512<T> Load512(ref T ptr) => Vector512.LoadUnsafe(ref ptr);
 471
 472            [MethodImpl(MethodImplOptions.AggressiveInlining)]
 473            [CompExactlyDependsOn(typeof(Avx))]
 474            public static bool EqualAndAscii256(ref T left, ref T right)
 475            {
 0476                Vector256<T> leftValues = Vector256.LoadUnsafe(ref left);
 0477                Vector256<T> rightValues = Vector256.LoadUnsafe(ref right);
 478
 0479                if (leftValues != rightValues || !AllCharsInVectorAreAscii(leftValues))
 480                {
 0481                    return false;
 482                }
 483
 0484                return true;
 485            }
 486
 487            [MethodImpl(MethodImplOptions.AggressiveInlining)]
 488            public static bool EqualAndAscii512(ref T left, ref T right)
 489            {
 0490                Vector512<T> leftValues = Vector512.LoadUnsafe(ref left);
 0491                Vector512<T> rightValues = Vector512.LoadUnsafe(ref right);
 492
 0493                if (leftValues != rightValues || !AllCharsInVectorAreAscii(leftValues))
 494                {
 0495                    return false;
 496                }
 497
 0498                return true;
 499            }
 500        }
 501
 502        private readonly struct WideningLoader : ILoader<byte, ushort>
 503        {
 504            [MethodImpl(MethodImplOptions.AggressiveInlining)]
 505            public static Vector128<ushort> Load128(ref byte ptr)
 506            {
 507                if (AdvSimd.IsSupported)
 508                {
 509                    return AdvSimd.ZeroExtendWideningLower(Vector64.LoadUnsafe(ref ptr));
 510                }
 0511                else if (Sse2.IsSupported)
 512                {
 0513                    Vector128<byte> vec = Vector128.CreateScalarUnsafe(Unsafe.ReadUnaligned<long>(ref ptr)).AsByte();
 0514                    return Sse2.UnpackLow(vec, Vector128<byte>.Zero).AsUInt16();
 515                }
 516                else if (PackedSimd.IsSupported)
 517                {
 518                    Vector128<byte> vec = Vector128.CreateScalarUnsafe(Unsafe.ReadUnaligned<long>(ref ptr)).AsByte();
 519                    return PackedSimd.ZeroExtendWideningLower(vec);
 520                }
 521                else
 522                {
 0523                    (Vector64<ushort> lower, Vector64<ushort> upper) = Vector64.Widen(Vector64.LoadUnsafe(ref ptr));
 0524                    return Vector128.Create(lower, upper);
 525                }
 526            }
 527
 528            [MethodImpl(MethodImplOptions.AggressiveInlining)]
 529            public static Vector256<ushort> Load256(ref byte ptr)
 530            {
 0531                (Vector128<ushort> lower, Vector128<ushort> upper) = Vector128.Widen(Vector128.LoadUnsafe(ref ptr));
 0532                return Vector256.Create(lower, upper);
 533            }
 534
 535            [MethodImpl(MethodImplOptions.AggressiveInlining)]
 536            public static Vector512<ushort> Load512(ref byte ptr)
 537            {
 0538                return Vector512.WidenLower(Vector256.LoadUnsafe(ref ptr).ToVector512());
 539            }
 540
 541            [MethodImpl(MethodImplOptions.AggressiveInlining)]
 542            [CompExactlyDependsOn(typeof(Avx))]
 543            public static bool EqualAndAscii256(ref byte utf8, ref ushort utf16)
 544            {
 545                // We widen the utf8 param so we can compare it to utf16, this doubles how much of the utf16 vector we s
 0546                Debug.Assert(Vector256<byte>.Count == Vector256<ushort>.Count * 2);
 547
 0548                Vector256<byte> leftNotWidened = Vector256.LoadUnsafe(ref utf8);
 0549                if (!AllCharsInVectorAreAscii(leftNotWidened))
 550                {
 0551                    return false;
 552                }
 553
 0554                (Vector256<ushort> leftLower, Vector256<ushort> leftUpper) = Vector256.Widen(leftNotWidened);
 0555                Vector256<ushort> right = Vector256.LoadUnsafe(ref utf16);
 0556                Vector256<ushort> rightNext = Vector256.LoadUnsafe(ref utf16, (uint)Vector256<ushort>.Count);
 557
 558                // A branchless version of "leftLower != right || leftUpper != rightNext"
 0559                if (((leftLower ^ right) | (leftUpper ^ rightNext)) != Vector256<ushort>.Zero)
 560                {
 0561                    return false;
 562                }
 563
 0564                return true;
 565            }
 566
 567            [MethodImpl(MethodImplOptions.AggressiveInlining)]
 568            public static bool EqualAndAscii512(ref byte utf8, ref ushort utf16)
 569            {
 570                // We widen the utf8 param so we can compare it to utf16, this doubles how much of the utf16 vector we s
 0571                Debug.Assert(Vector512<byte>.Count == Vector512<ushort>.Count * 2);
 572
 0573                Vector512<byte> leftNotWidened = Vector512.LoadUnsafe(ref utf8);
 0574                if (!AllCharsInVectorAreAscii(leftNotWidened))
 575                {
 0576                    return false;
 577                }
 578
 0579                (Vector512<ushort> leftLower, Vector512<ushort> leftUpper) = Vector512.Widen(leftNotWidened);
 0580                Vector512<ushort> right = Vector512.LoadUnsafe(ref utf16);
 0581                Vector512<ushort> rightNext = Vector512.LoadUnsafe(ref utf16, (uint)Vector512<ushort>.Count);
 582
 583                // A branchless version of "leftLower != right || leftUpper != rightNext"
 0584                if (((leftLower ^ right) | (leftUpper ^ rightNext)) != Vector512<ushort>.Zero)
 585                {
 0586                    return false;
 587                }
 588
 0589                return true;
 590            }
 591        }
 592    }
 593}
 594

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Transcoding.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Buffers;
 5using System.Diagnostics;
 6using System.Runtime.InteropServices;
 7
 8namespace System.Text
 9{
 10    public static partial class Ascii
 11    {
 12        /// <summary>
 13        /// Copies text from a source buffer to a destination buffer, converting
 14        /// from ASCII to UTF-16 during the copy.
 15        /// </summary>
 16        /// <param name="source">The source buffer from which ASCII text is read.</param>
 17        /// <param name="destination">The destination buffer to which UTF-16 text is written.</param>
 18        /// <param name="charsWritten">The number of chars actually written to <paramref name="destination"/>. It's the 
 19        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 20        public static unsafe OperationStatus ToUtf16(ReadOnlySpan<byte> source, Span<char> destination, out int charsWri
 21        {
 22            nuint numElementsToConvert;
 23            OperationStatus statusToReturnOnSuccess;
 24
 201478925            if (source.Length <= destination.Length)
 26            {
 201478927                numElementsToConvert = (uint)source.Length;
 201478928                statusToReturnOnSuccess = OperationStatus.Done;
 29            }
 30            else
 31            {
 032                numElementsToConvert = (uint)destination.Length;
 033                statusToReturnOnSuccess = OperationStatus.DestinationTooSmall;
 34            }
 35
 201478936            fixed (byte* pSource = &MemoryMarshal.GetReference(source))
 201478937            fixed (char* pDestination = &MemoryMarshal.GetReference(destination))
 38            {
 201478939                nuint numElementsActuallyConverted = WidenAsciiToUtf16(pSource, pDestination, numElementsToConvert);
 201478940                Debug.Assert(numElementsActuallyConverted <= numElementsToConvert);
 41
 201478942                charsWritten = (int)numElementsActuallyConverted;
 201478943                return (numElementsToConvert == numElementsActuallyConverted) ? statusToReturnOnSuccess : OperationStatu
 44            }
 45        }
 46
 47        /// <summary>
 48        /// Copies text from a source buffer to a destination buffer, converting
 49        /// from UTF-16 to ASCII during the copy.
 50        /// </summary>
 51        /// <param name="source">The source buffer from which UTF-16 text is read.</param>
 52        /// <param name="destination">The destination buffer to which ASCII text is written.</param>
 53        /// <param name="bytesWritten">The number of bytes actually written to <paramref name="destination"/>. It's the 
 54        /// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
 55        public static unsafe OperationStatus FromUtf16(ReadOnlySpan<char> source, Span<byte> destination, out int bytesW
 56        {
 57            nuint numElementsToConvert;
 58            OperationStatus statusToReturnOnSuccess;
 59
 060            if (source.Length <= destination.Length)
 61            {
 062                numElementsToConvert = (uint)source.Length;
 063                statusToReturnOnSuccess = OperationStatus.Done;
 64            }
 65            else
 66            {
 067                numElementsToConvert = (uint)destination.Length;
 068                statusToReturnOnSuccess = OperationStatus.DestinationTooSmall;
 69            }
 70
 071            fixed (char* pSource = &MemoryMarshal.GetReference(source))
 072            fixed (byte* pDestination = &MemoryMarshal.GetReference(destination))
 73            {
 074                nuint numElementsActuallyConverted = NarrowUtf16ToAscii(pSource, pDestination, numElementsToConvert);
 075                Debug.Assert(numElementsActuallyConverted <= numElementsToConvert);
 76
 077                bytesWritten = (int)numElementsActuallyConverted;
 078                return (numElementsToConvert == numElementsActuallyConverted) ? statusToReturnOnSuccess : OperationStatu
 79            }
 80        }
 81    }
 82}
 83

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Trimming.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Numerics;
 5
 6namespace System.Text
 7{
 8    public static partial class Ascii
 9    {
 10        /// <summary>
 11        /// Trims all leading and trailing ASCII whitespaces from the buffer.
 12        /// </summary>
 13        /// <param name="value">The ASCII buffer.</param>
 14        /// <returns>The Range of the untrimmed data.</returns>
 015        public static Range Trim(ReadOnlySpan<byte> value) => TrimHelper(value, TrimType.Both);
 16
 17        /// <inheritdoc cref="Trim(ReadOnlySpan{byte})"/>
 018        public static Range Trim(ReadOnlySpan<char> value) => TrimHelper(value, TrimType.Both);
 19
 20        /// <summary>
 21        /// Trims all leading ASCII whitespaces from the buffer.
 22        /// </summary>
 23        /// <param name="value">The ASCII buffer.</param>
 24        /// <returns>The Range of the untrimmed data.</returns>
 025        public static Range TrimStart(ReadOnlySpan<byte> value) => TrimHelper(value, TrimType.Head);
 26
 27        /// <inheritdoc cref="TrimStart(ReadOnlySpan{byte})"/>
 028        public static Range TrimStart(ReadOnlySpan<char> value) => TrimHelper(value, TrimType.Head);
 29
 30        /// <summary>
 31        /// Trims all trailing ASCII whitespaces from the buffer.
 32        /// </summary>
 33        /// <param name="value">The ASCII buffer.</param>
 34        /// <returns>The Range of the untrimmed data.</returns>
 035        public static Range TrimEnd(ReadOnlySpan<byte> value) => TrimHelper(value, TrimType.Tail);
 36
 37        /// <inheritdoc cref="TrimEnd(ReadOnlySpan{byte})"/>
 038        public static Range TrimEnd(ReadOnlySpan<char> value) => TrimHelper(value, TrimType.Tail);
 39
 40        private static Range TrimHelper<T>(ReadOnlySpan<T> value, TrimType trimType)
 41            where T : unmanaged, IBinaryInteger<T>
 42        {
 43            // A bitmap with a bit set for each ASCII whitespace character. The set bit is at the
 44            // index of the character minus 1, since we're using a 32-bit value and space would otherwise
 45            // be at index 32; with -1, it's at index 31.
 46            const uint TrimMask =
 47                  (1u << (0x09 - 1))
 48                | (1u << (0x0A - 1))
 49                | (1u << (0x0B - 1))
 50                | (1u << (0x0C - 1))
 51                | (1u << (0x0D - 1))
 52                | (1u << (0x20 - 1));
 53
 054            int start = 0;
 055            if ((trimType & TrimType.Head) != 0)
 56            {
 057                for (; start < value.Length; start++)
 58                {
 059                    uint elementValueM1 = uint.CreateTruncating(value[start]) - 1;
 060                    if ((elementValueM1 > 0x1F) || ((TrimMask & (1u << ((int)elementValueM1))) == 0))
 61                    {
 62                        break;
 63                    }
 64                }
 65            }
 66
 067            int end = value.Length - 1;
 068            if ((trimType & TrimType.Tail) != 0)
 69            {
 070                for (; start <= end; end--)
 71                {
 072                    uint elementValueM1 = uint.CreateTruncating(value[end]) - 1;
 073                    if ((elementValueM1 > 0x1F) || ((TrimMask & (1u << ((int)elementValueM1))) == 0))
 74                    {
 75                        break;
 76                    }
 77                }
 78            }
 79
 080            return start..(end + 1);
 81        }
 82    }
 83}
 84

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Diagnostics;
 5using System.Diagnostics.CodeAnalysis;
 6using System.Numerics;
 7using System.Runtime.CompilerServices;
 8#if NET
 9using System.Runtime.Intrinsics;
 10using System.Runtime.Intrinsics.Arm;
 11using System.Runtime.Intrinsics.Wasm;
 12using System.Runtime.Intrinsics.X86;
 13#endif
 14
 15namespace System.Text
 16{
 17#if SYSTEM_PRIVATE_CORELIB
 18    public
 19#else
 20    internal
 21#endif
 22        static partial class Ascii
 23    {
 24        /// <summary>
 25        /// Returns <see langword="true"/> iff all bytes in <paramref name="value"/> are ASCII.
 26        /// </summary>
 27        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 28        private static bool AllBytesInUInt64AreAscii(ulong value)
 29        {
 30            // If the high bit of any byte is set, that byte is non-ASCII.
 31
 032            return (value & UInt64HighBitsOnlyMask) == 0;
 33        }
 34
 35        /// <summary>
 36        /// Returns <see langword="true"/> iff all chars in <paramref name="value"/> are ASCII.
 37        /// </summary>
 38        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 39        private static bool AllCharsInUInt32AreAscii(uint value)
 40        {
 1114741            return (value & ~0x007F007Fu) == 0;
 42        }
 43
 44        /// <summary>
 45        /// Returns <see langword="true"/> iff all chars in <paramref name="value"/> are ASCII.
 46        /// </summary>
 47        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 48        private static bool AllCharsInUInt64AreAscii(ulong value)
 49        {
 3434850            return (value & ~0x007F007F_007F007Ful) == 0;
 51        }
 52
 53        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 54        private static bool AllCharsInUInt64AreAscii<T>(ulong value)
 55            where T : unmanaged
 56        {
 157            Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));
 58
 159            return typeof(T) == typeof(byte)
 160                ? AllBytesInUInt64AreAscii(value)
 161                : AllCharsInUInt64AreAscii(value);
 62        }
 63
 64#if NET
 65        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 66        [CompExactlyDependsOn(typeof(AdvSimd.Arm64))]
 67        private static int GetIndexOfFirstNonAsciiByteInLane_AdvSimd(Vector128<byte> value, Vector128<byte> bitmask)
 68        {
 69            if (!AdvSimd.Arm64.IsSupported || !BitConverter.IsLittleEndian)
 70            {
 71                throw new PlatformNotSupportedException();
 72            }
 73
 74            // extractedBits[i] = (value[i] >> 7) & (1 << (12 * (i % 2)));
 75            Vector128<byte> mostSignificantBitIsSet = (value.AsSByte() >> 7).AsByte();
 76            Vector128<byte> extractedBits = mostSignificantBitIsSet & bitmask;
 77
 78            // collapse mask to lower bits
 79            extractedBits = AdvSimd.Arm64.AddPairwise(extractedBits, extractedBits);
 80            ulong mask = extractedBits.AsUInt64().ToScalar();
 81
 82            // calculate the index
 83            int index = BitOperations.TrailingZeroCount(mask) >> 2;
 84            Debug.Assert((mask != 0) ? index < 16 : index >= 16);
 85            return index;
 86        }
 87#endif
 88
 89        /// <summary>
 90        /// Given a DWORD which represents two packed chars in machine-endian order,
 91        /// <see langword="true"/> iff the first char (in machine-endian order) is ASCII.
 92        /// </summary>
 93        /// <param name="value"></param>
 94        /// <returns></returns>
 95        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 96        private static bool FirstCharInUInt32IsAscii(uint value)
 97        {
 98            return (BitConverter.IsLittleEndian && (value & 0xFF80u) == 0)
 99                || (!BitConverter.IsLittleEndian && (value & 0xFF800000u) == 0);
 100        }
 101
 102        /// <summary>
 103        /// Returns the index in <paramref name="pBuffer"/> where the first non-ASCII byte is found.
 104        /// Returns <paramref name="bufferLength"/> if the buffer is empty or all-ASCII.
 105        /// </summary>
 106        /// <returns>An ASCII byte is defined as 0x00 - 0x7F, inclusive.</returns>
 107        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 108        internal static unsafe nuint GetIndexOfFirstNonAsciiByte(byte* pBuffer, nuint bufferLength)
 109        {
 110            // If 256/512-bit aren't supported but SSE2 is supported, use those specific intrinsics instead of
 111            // the generic vectorized code. This has two benefits: (a) we can take advantage of specific instructions
 112            // like pmovmskb which we know are optimized, and (b) we can avoid downclocking the processor while
 113            // this method is running.
 114
 115#if NET
 855076116            if (!Vector512.IsHardwareAccelerated &&
 855076117                !Vector256.IsHardwareAccelerated &&
 855076118                (Sse2.IsSupported || AdvSimd.IsSupported))
 119            {
 0120                return GetIndexOfFirstNonAsciiByte_Intrinsified(pBuffer, bufferLength);
 121            }
 122            else
 123#endif
 124            {
 125                // Handles Vector512, Vector256, Vector128, and scalar.
 855076126                return GetIndexOfFirstNonAsciiByte_Vector(pBuffer, bufferLength);
 127            }
 128        }
 129
 130        private static unsafe nuint GetIndexOfFirstNonAsciiByte_Vector(byte* pBuffer, nuint bufferLength)
 131        {
 132            // Squirrel away the original buffer reference. This method works by determining the exact
 133            // byte reference where non-ASCII data begins, so we need this base value to perform the
 134            // final subtraction at the end of the method to get the index into the original buffer.
 135
 136            byte* pOriginalBuffer = pBuffer;
 137
 138            // Before we drain off byte-by-byte, try a generic vectorized loop.
 139            // Only run the loop if we have at least two vectors we can pull out.
 140            // Note use of SBYTE instead of BYTE below; we're using the two's-complement
 141            // representation of negative integers to act as a surrogate for "is ASCII?".
 142
 143#if NET
 855076144            if (Vector512.IsHardwareAccelerated && bufferLength >= 2 * (uint)Vector512<byte>.Count)
 145            {
 295304146                if (Vector512.Load(pBuffer).ExtractMostSignificantBits() == 0)
 147                {
 148                    // The first several elements of the input buffer were ASCII. Bump up the pointer to the
 149                    // next aligned boundary, then perform aligned reads from here on out until we find non-ASCII
 150                    // data or we approach the end of the buffer. It's possible we'll reread data; this is ok.
 151
 1396152                    byte* pFinalVectorReadPos = pBuffer + bufferLength - Vector512.Size;
 1396153                    pBuffer = (byte*)(((nuint)pBuffer + Vector512.Size) & ~(nuint)(Vector512.Size - 1));
 154
 155#if DEBUG
 1396156                    long numBytesRead = pBuffer - pOriginalBuffer;
 1396157                    Debug.Assert(0 < numBytesRead && numBytesRead <= Vector512.Size, "We should've made forward progress
 1396158                    Debug.Assert((nuint)numBytesRead <= bufferLength, "We shouldn't have read past the end of the input 
 159#endif
 160
 1396161                    Debug.Assert(pBuffer <= pFinalVectorReadPos, "Should be able to read at least one vector.");
 162
 163                    do
 164                    {
 3368165                        Debug.Assert((nuint)pBuffer % Vector512.Size == 0, "Vector read should be aligned.");
 3368166                        if (Vector512.LoadAligned(pBuffer).ExtractMostSignificantBits() != 0)
 167                        {
 168                            break; // found non-ASCII data
 169                        }
 170
 2361171                        pBuffer += Vector512.Size;
 2361172                    } while (pBuffer <= pFinalVectorReadPos);
 173
 174                    // Adjust the remaining buffer length for the number of elements we just consumed.
 175
 1396176                    bufferLength -= (nuint)pBuffer;
 1396177                    bufferLength += (nuint)pOriginalBuffer;
 178                }
 179            }
 559772180            else if (Vector256.IsHardwareAccelerated && bufferLength >= 2 * (uint)Vector256<byte>.Count)
 181            {
 163342182                if (Vector256.Load(pBuffer).ExtractMostSignificantBits() == 0)
 183                {
 184                    // The first several elements of the input buffer were ASCII. Bump up the pointer to the
 185                    // next aligned boundary, then perform aligned reads from here on out until we find non-ASCII
 186                    // data or we approach the end of the buffer. It's possible we'll reread data; this is ok.
 187
 1023188                    byte* pFinalVectorReadPos = pBuffer + bufferLength - Vector256.Size;
 1023189                    pBuffer = (byte*)(((nuint)pBuffer + Vector256.Size) & ~(nuint)(Vector256.Size - 1));
 190
 191#if DEBUG
 1023192                    long numBytesRead = pBuffer - pOriginalBuffer;
 1023193                    Debug.Assert(0 < numBytesRead && numBytesRead <= Vector256.Size, "We should've made forward progress
 1023194                    Debug.Assert((nuint)numBytesRead <= bufferLength, "We shouldn't have read past the end of the input 
 195#endif
 196
 1023197                    Debug.Assert(pBuffer <= pFinalVectorReadPos, "Should be able to read at least one vector.");
 198
 199                    do
 200                    {
 2147201                        Debug.Assert((nuint)pBuffer % Vector256.Size == 0, "Vector read should be aligned.");
 2147202                        if (Vector256.LoadAligned(pBuffer).ExtractMostSignificantBits() != 0)
 203                        {
 204                            break; // found non-ASCII data
 205                        }
 206
 1440207                        pBuffer += Vector256.Size;
 1440208                    } while (pBuffer <= pFinalVectorReadPos);
 209
 210                    // Adjust the remaining buffer length for the number of elements we just consumed.
 211
 1023212                    bufferLength -= (nuint)pBuffer;
 1023213                    bufferLength += (nuint)pOriginalBuffer;
 214                }
 215            }
 396430216            else if (Vector128.IsHardwareAccelerated && bufferLength >= 2 * (uint)Vector128<byte>.Count)
 217            {
 143031218                if (!VectorContainsNonAsciiChar(Vector128.Load(pBuffer)))
 219                {
 220                    // The first several elements of the input buffer were ASCII. Bump up the pointer to the
 221                    // next aligned boundary, then perform aligned reads from here on out until we find non-ASCII
 222                    // data or we approach the end of the buffer. It's possible we'll reread data; this is ok.
 223
 1587224                    byte* pFinalVectorReadPos = pBuffer + bufferLength - Vector128.Size;
 1587225                    pBuffer = (byte*)(((nuint)pBuffer + Vector128.Size) & ~(nuint)(Vector128.Size - 1));
 226
 227#if DEBUG
 1587228                    long numBytesRead = pBuffer - pOriginalBuffer;
 1587229                    Debug.Assert(0 < numBytesRead && numBytesRead <= Vector128.Size, "We should've made forward progress
 1587230                    Debug.Assert((nuint)numBytesRead <= bufferLength, "We shouldn't have read past the end of the input 
 231#endif
 232
 1587233                    Debug.Assert(pBuffer <= pFinalVectorReadPos, "Should be able to read at least one vector.");
 234
 235                    do
 236                    {
 2901237                        Debug.Assert((nuint)pBuffer % Vector128.Size == 0, "Vector read should be aligned.");
 2901238                        if (VectorContainsNonAsciiChar(Vector128.LoadAligned(pBuffer)))
 239                        {
 240                            break; // found non-ASCII data
 241                        }
 242
 1730243                        pBuffer += Vector128.Size;
 1730244                    } while (pBuffer <= pFinalVectorReadPos);
 245
 246                    // Adjust the remaining buffer length for the number of elements we just consumed.
 247
 1587248                    bufferLength -= (nuint)pBuffer;
 1587249                    bufferLength += (nuint)pOriginalBuffer;
 250                }
 251            }
 252#endif
 253
 254            // At this point, the buffer length wasn't enough to perform a vectorized search, or we did perform
 255            // a vectorized search and encountered non-ASCII data. In either case go down a non-vectorized code
 256            // path to drain any remaining ASCII bytes.
 257            //
 258            // We're going to perform unaligned reads, so prefer 32-bit reads instead of 64-bit reads.
 259            // This also allows us to perform more optimized bit twiddling tricks to count the number of ASCII bytes.
 260
 261            uint currentUInt32;
 262
 263            // Try reading 64 bits at a time in a loop.
 264
 905432265            for (; bufferLength >= 8; bufferLength -= 8)
 266            {
 796692267                currentUInt32 = Unsafe.ReadUnaligned<uint>(pBuffer);
 796692268                uint nextUInt32 = Unsafe.ReadUnaligned<uint>(pBuffer + 4);
 269
 796692270                if (!AllBytesInUInt32AreAscii(currentUInt32 | nextUInt32))
 271                {
 272                    // One of these two values contains non-ASCII bytes.
 273                    // Figure out which one it is, then put it in 'current' so that we can drain the ASCII bytes.
 274
 771514275                    if (AllBytesInUInt32AreAscii(currentUInt32))
 276                    {
 25325277                        currentUInt32 = nextUInt32;
 25325278                        pBuffer += 4;
 279                    }
 280
 25325281                    goto FoundNonAsciiData;
 282                }
 283
 25178284                pBuffer += 8; // consumed 8 ASCII bytes
 285            }
 286
 287            // From this point forward we don't need to update bufferLength.
 288            // Try reading 32 bits.
 289
 83562290            if ((bufferLength & 4) != 0)
 291            {
 44247292                currentUInt32 = Unsafe.ReadUnaligned<uint>(pBuffer);
 44247293                if (!AllBytesInUInt32AreAscii(currentUInt32))
 294                {
 295                    goto FoundNonAsciiData;
 296                }
 297
 3765298                pBuffer += 4;
 299            }
 300
 301            // Try reading 16 bits.
 302
 43080303            if ((bufferLength & 2) != 0)
 304            {
 26894305                currentUInt32 = Unsafe.ReadUnaligned<ushort>(pBuffer);
 26894306                if (!AllBytesInUInt32AreAscii(currentUInt32))
 307                {
 22665308                    if (!BitConverter.IsLittleEndian)
 309                    {
 310                        currentUInt32 <<= 16;
 311                    }
 312                    goto FoundNonAsciiData;
 313                }
 314
 4229315                pBuffer += 2;
 316            }
 317
 318            // Try reading 8 bits
 319
 20415320            if ((bufferLength & 1) != 0)
 321            {
 322                // If the buffer contains non-ASCII data, the comparison below will fail, and
 323                // we'll end up not incrementing the buffer reference.
 324
 16379325                if (*(sbyte*)pBuffer >= 0)
 326                {
 5850327                    pBuffer++;
 328                }
 329            }
 330
 331        Finish:
 332
 855076333            nuint totalNumBytesRead = (nuint)pBuffer - (nuint)pOriginalBuffer;
 855076334            return totalNumBytesRead;
 335
 336        FoundNonAsciiData:
 337
 834661338            Debug.Assert(!AllBytesInUInt32AreAscii(currentUInt32), "Shouldn't have reached this point if we have an all-
 339
 340            // The method being called doesn't bother looking at whether the high byte is ASCII. There are only
 341            // two scenarios: (a) either one of the earlier bytes is not ASCII and the search terminates before
 342            // we get to the high byte; or (b) all of the earlier bytes are ASCII, so the high byte must be
 343            // non-ASCII. In both cases we only care about the low 24 bits.
 344
 834661345            pBuffer += CountNumberOfLeadingAsciiBytesFromUInt32WithSomeNonAsciiData(currentUInt32);
 834661346            goto Finish;
 347        }
 348
 349#if NET
 350        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 351        private static bool ContainsNonAsciiByte_Sse2(uint sseMask)
 352        {
 0353            Debug.Assert(sseMask != uint.MaxValue);
 0354            Debug.Assert(Sse2.IsSupported);
 0355            return sseMask != 0;
 356        }
 357
 358        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 359        private static bool ContainsNonAsciiByte_AdvSimd(uint advSimdIndex)
 360        {
 361            Debug.Assert(advSimdIndex != uint.MaxValue);
 362            Debug.Assert(AdvSimd.IsSupported);
 363            return advSimdIndex < 16;
 364        }
 365
 366        private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuffer, nuint bufferLength)
 367        {
 368            // JIT turns the below into constants
 369
 370            uint SizeOfVector128 = (uint)sizeof(Vector128<byte>);
 0371            nuint MaskOfAllBitsInVector128 = (nuint)(SizeOfVector128 - 1);
 372
 0373            Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported, "Sse2 or AdvSimd64 required.");
 0374            Debug.Assert(BitConverter.IsLittleEndian, "This SSE2/Arm64 implementation assumes little-endian.");
 375
 0376            Vector128<byte> bitmask = BitConverter.IsLittleEndian ?
 0377                Vector128.Create((ushort)0x1001).AsByte() :
 0378                Vector128.Create((ushort)0x0110).AsByte();
 379
 0380            uint currentSseMask = uint.MaxValue, secondSseMask = uint.MaxValue;
 0381            uint currentAdvSimdIndex = uint.MaxValue, secondAdvSimdIndex = uint.MaxValue;
 0382            byte* pOriginalBuffer = pBuffer;
 383
 384            // This method is written such that control generally flows top-to-bottom, avoiding
 385            // jumps as much as possible in the optimistic case of a large enough buffer and
 386            // "all ASCII". If we see non-ASCII data, we jump out of the hot paths to targets
 387            // after all the main logic.
 388
 0389            if (bufferLength < SizeOfVector128)
 390            {
 391                goto InputBufferLessThanOneVectorInLength; // can't vectorize; drain primitives instead
 392            }
 393
 394            // Read the first vector unaligned.
 395
 0396            if (Sse2.IsSupported)
 397            {
 0398                currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load
 0399                if (ContainsNonAsciiByte_Sse2(currentSseMask))
 400                {
 0401                    goto FoundNonAsciiDataInCurrentChunk;
 402                }
 403            }
 404            else if (AdvSimd.Arm64.IsSupported)
 405            {
 406                Vector128<byte> vector = AdvSimd.LoadVector128(pBuffer);
 407                if (VectorContainsNonAsciiChar(vector))
 408                {
 409                    currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(vector, bitmask); // unaligned
 410                    goto FoundNonAsciiDataInCurrentChunk;
 411                }
 412            }
 413            else
 414            {
 0415                throw new PlatformNotSupportedException();
 416            }
 417
 418            // If we have less than 32 bytes to process, just go straight to the final unaligned
 419            // read. There's no need to mess with the loop logic in the middle of this method.
 420
 0421            if (bufferLength < 2 * SizeOfVector128)
 422            {
 423                goto IncrementCurrentOffsetBeforeFinalUnalignedVectorRead;
 424            }
 425
 426            // Now adjust the read pointer so that future reads are aligned.
 427
 0428            pBuffer = (byte*)(((nuint)pBuffer + SizeOfVector128) & ~(nuint)MaskOfAllBitsInVector128);
 429
 430#if DEBUG
 0431            long numBytesRead = pBuffer - pOriginalBuffer;
 0432            Debug.Assert(0 < numBytesRead && numBytesRead <= SizeOfVector128, "We should've made forward progress of at 
 0433            Debug.Assert((nuint)numBytesRead <= bufferLength, "We shouldn't have read past the end of the input buffer."
 434#endif
 435
 436            // Adjust the remaining length to account for what we just read.
 437
 0438            bufferLength += (nuint)pOriginalBuffer;
 0439            bufferLength -= (nuint)pBuffer;
 440
 441            // The buffer is now properly aligned.
 442            // Read 2 vectors at a time if possible.
 443
 0444            if (bufferLength >= 2 * SizeOfVector128)
 445            {
 0446                byte* pFinalVectorReadPos = (byte*)((nuint)pBuffer + bufferLength - 2 * SizeOfVector128);
 447
 448                // After this point, we no longer need to update the bufferLength value.
 449
 450                do
 451                {
 0452                    if (Sse2.IsSupported)
 453                    {
 0454                        Vector128<byte> firstVector = Sse2.LoadAlignedVector128(pBuffer);
 0455                        Vector128<byte> secondVector = Sse2.LoadAlignedVector128(pBuffer + SizeOfVector128);
 456
 0457                        currentSseMask = (uint)Sse2.MoveMask(firstVector);
 0458                        secondSseMask = (uint)Sse2.MoveMask(secondVector);
 0459                        if (ContainsNonAsciiByte_Sse2(currentSseMask | secondSseMask))
 460                        {
 0461                            goto FoundNonAsciiDataInInnerLoop;
 462                        }
 463                    }
 464                    else if (AdvSimd.Arm64.IsSupported)
 465                    {
 466                        Vector128<byte> firstVector = AdvSimd.LoadVector128(pBuffer);
 467                        Vector128<byte> secondVector = AdvSimd.LoadVector128(pBuffer + SizeOfVector128);
 468
 469                        if (VectorContainsNonAsciiChar(firstVector | secondVector))
 470                        {
 471                            currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(firstVector, bitmask);
 472                            secondAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(secondVector, bitmask);
 473                            goto FoundNonAsciiDataInInnerLoop;
 474                        }
 475                    }
 476                    else
 477                    {
 0478                        throw new PlatformNotSupportedException();
 479                    }
 480
 0481                    pBuffer += 2 * SizeOfVector128;
 0482                } while (pBuffer <= pFinalVectorReadPos);
 483            }
 484
 485            // We have somewhere between 0 and (2 * vector length) - 1 bytes remaining to read from.
 486            // Since the above loop doesn't update bufferLength, we can't rely on its absolute value.
 487            // But we _can_ rely on it to tell us how much remaining data must be drained by looking
 488            // at what bits of it are set. This works because had we updated it within the loop above,
 489            // we would've been adding 2 * SizeOfVector128 on each iteration, but we only care about
 490            // bits which are less significant than those that the addition would've acted on.
 491
 492            // If there is fewer than one vector length remaining, skip the next aligned read.
 493
 0494            if ((bufferLength & SizeOfVector128) == 0)
 495            {
 496                goto DoFinalUnalignedVectorRead;
 497            }
 498
 499            // At least one full vector's worth of data remains, so we can safely read it.
 500            // Remember, at this point pBuffer is still aligned.
 501
 0502            if (Sse2.IsSupported)
 503            {
 0504                currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer));
 0505                if (ContainsNonAsciiByte_Sse2(currentSseMask))
 506                {
 0507                    goto FoundNonAsciiDataInCurrentChunk;
 508                }
 509            }
 510            else if (AdvSimd.Arm64.IsSupported)
 511            {
 512                Vector128<byte> vector = AdvSimd.LoadVector128(pBuffer);
 513                if (VectorContainsNonAsciiChar(vector))
 514                {
 515                    currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(vector, bitmask);
 516                    goto FoundNonAsciiDataInCurrentChunk;
 517                }
 518            }
 519            else
 520            {
 0521                throw new PlatformNotSupportedException();
 522            }
 523
 524        IncrementCurrentOffsetBeforeFinalUnalignedVectorRead:
 525
 0526            pBuffer += SizeOfVector128;
 527
 528        DoFinalUnalignedVectorRead:
 529
 0530            if (((byte)bufferLength & MaskOfAllBitsInVector128) != 0)
 531            {
 532                // Perform an unaligned read of the last vector.
 533                // We need to adjust the pointer because we're re-reading data.
 534
 0535                pBuffer += (bufferLength & MaskOfAllBitsInVector128) - SizeOfVector128;
 536
 0537                if (Sse2.IsSupported)
 538                {
 0539                    currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load
 0540                    if (ContainsNonAsciiByte_Sse2(currentSseMask))
 541                    {
 0542                        goto FoundNonAsciiDataInCurrentChunk;
 543                    }
 544
 545                }
 546                else if (AdvSimd.Arm64.IsSupported)
 547                {
 548                    Vector128<byte> vector = AdvSimd.LoadVector128(pBuffer);
 549                    if (VectorContainsNonAsciiChar(vector))
 550                    {
 551                        currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(vector, bitmask); // unali
 552                        goto FoundNonAsciiDataInCurrentChunk;
 553                    }
 554
 555                }
 556                else
 557                {
 0558                    throw new PlatformNotSupportedException();
 559                }
 560
 0561                pBuffer += SizeOfVector128;
 562            }
 563
 564        Finish:
 0565            return (nuint)pBuffer - (nuint)pOriginalBuffer; // and we're done!
 566
 567        FoundNonAsciiDataInInnerLoop:
 568
 569            // If the current (first) mask isn't the mask that contains non-ASCII data, then it must
 570            // instead be the second mask. If so, skip the entire first mask and drain ASCII bytes
 571            // from the second mask.
 572
 0573            if (Sse2.IsSupported)
 574            {
 0575                if (!ContainsNonAsciiByte_Sse2(currentSseMask))
 576                {
 0577                    pBuffer += SizeOfVector128;
 0578                    currentSseMask = secondSseMask;
 579                }
 580            }
 581            else if (AdvSimd.IsSupported)
 582            {
 583                if (!ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex))
 584                {
 585                    pBuffer += SizeOfVector128;
 586                    currentAdvSimdIndex = secondAdvSimdIndex;
 587                }
 588            }
 589            else
 590            {
 0591                throw new PlatformNotSupportedException();
 592            }
 593        FoundNonAsciiDataInCurrentChunk:
 594
 595
 0596            if (Sse2.IsSupported)
 597            {
 598                // The mask contains - from the LSB - a 0 for each ASCII byte we saw, and a 1 for each non-ASCII byte.
 599                // Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't
 600                // available, we'll fall back to a normal loop.
 0601                Debug.Assert(ContainsNonAsciiByte_Sse2(currentSseMask), "Shouldn't be here unless we see non-ASCII data.
 0602                pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMask);
 603            }
 604            else if (AdvSimd.Arm64.IsSupported)
 605            {
 606                Debug.Assert(ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex), "Shouldn't be here unless we see non-ASC
 607                pBuffer += currentAdvSimdIndex;
 608            }
 609            else
 610            {
 0611                throw new PlatformNotSupportedException();
 612            }
 613
 614            goto Finish;
 615
 616        FoundNonAsciiDataInCurrentDWord:
 617
 618            uint currentDWord;
 0619            Debug.Assert(!AllBytesInUInt32AreAscii(currentDWord), "Shouldn't be here unless we see non-ASCII data.");
 0620            pBuffer += CountNumberOfLeadingAsciiBytesFromUInt32WithSomeNonAsciiData(currentDWord);
 621
 0622            goto Finish;
 623
 624        InputBufferLessThanOneVectorInLength:
 625
 626            // These code paths get hit if the original input length was less than one vector in size.
 627            // We can't perform vectorized reads at this point, so we'll fall back to reading primitives
 628            // directly. Note that all of these reads are unaligned.
 629
 0630            Debug.Assert(bufferLength < SizeOfVector128);
 631
 632            // QWORD drain
 633
 0634            if ((bufferLength & 8) != 0)
 635            {
 636                if (UIntPtr.Size == sizeof(ulong))
 637                {
 638                    // If we can use 64-bit tzcnt to count the number of leading ASCII bytes, prefer it.
 639
 0640                    ulong candidateUInt64 = Unsafe.ReadUnaligned<ulong>(pBuffer);
 0641                    if (!AllBytesInUInt64AreAscii(candidateUInt64))
 642                    {
 643                        // Clear everything but the high bit of each byte, then tzcnt.
 644                        // Remember to divide by 8 at the end to convert bit count to byte count.
 645
 0646                        candidateUInt64 &= UInt64HighBitsOnlyMask;
 0647                        pBuffer += (nuint)(BitOperations.TrailingZeroCount(candidateUInt64) >> 3);
 0648                        goto Finish;
 649                    }
 650                }
 651                else
 652                {
 653                    // If we can't use 64-bit tzcnt, no worries. We'll just do 2x 32-bit reads instead.
 654
 655                    currentDWord = Unsafe.ReadUnaligned<uint>(pBuffer);
 656                    uint nextDWord = Unsafe.ReadUnaligned<uint>(pBuffer + 4);
 657
 658                    if (!AllBytesInUInt32AreAscii(currentDWord | nextDWord))
 659                    {
 660                        // At least one of the values wasn't all-ASCII.
 661                        // We need to figure out which one it was and stick it in the currentMask local.
 662
 663                        if (AllBytesInUInt32AreAscii(currentDWord))
 664                        {
 665                            currentDWord = nextDWord; // this one is the culprit
 666                            pBuffer += 4;
 667                        }
 668
 669                        goto FoundNonAsciiDataInCurrentDWord;
 670                    }
 671                }
 672
 0673                pBuffer += 8; // successfully consumed 8 ASCII bytes
 674            }
 675
 676            // DWORD drain
 677
 0678            if ((bufferLength & 4) != 0)
 679            {
 0680                currentDWord = Unsafe.ReadUnaligned<uint>(pBuffer);
 681
 0682                if (!AllBytesInUInt32AreAscii(currentDWord))
 683                {
 684                    goto FoundNonAsciiDataInCurrentDWord;
 685                }
 686
 0687                pBuffer += 4; // successfully consumed 4 ASCII bytes
 688            }
 689
 690            // WORD drain
 691            // (We movzx to a DWORD for ease of manipulation.)
 692
 0693            if ((bufferLength & 2) != 0)
 694            {
 0695                currentDWord = Unsafe.ReadUnaligned<ushort>(pBuffer);
 696
 0697                if (!AllBytesInUInt32AreAscii(currentDWord))
 698                {
 699                    // We only care about the 0x0080 bit of the value. If it's not set, then we
 700                    // increment currentOffset by 1. If it's set, we don't increment it at all.
 701
 0702                    pBuffer += (nuint)((nint)(sbyte)currentDWord >> 7) + 1;
 0703                    goto Finish;
 704                }
 705
 0706                pBuffer += 2; // successfully consumed 2 ASCII bytes
 707            }
 708
 709            // BYTE drain
 710
 0711            if ((bufferLength & 1) != 0)
 712            {
 713                // sbyte has non-negative value if byte is ASCII.
 714
 0715                if (*(sbyte*)(pBuffer) >= 0)
 716                {
 0717                    pBuffer++; // successfully consumed a single byte
 718                }
 719            }
 720
 0721            goto Finish;
 722        }
 723#endif
 724
 725        /// <summary>
 726        /// Returns the index in <paramref name="pBuffer"/> where the first non-ASCII char is found.
 727        /// Returns <paramref name="bufferLength"/> if the buffer is empty or all-ASCII.
 728        /// </summary>
 729        /// <returns>An ASCII char is defined as 0x0000 - 0x007F, inclusive.</returns>
 730        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 731        internal static unsafe nuint GetIndexOfFirstNonAsciiChar(char* pBuffer, nuint bufferLength /* in chars */)
 732        {
 733            // If 256/512-bit aren't supported but SSE2/ASIMD is supported, use those specific intrinsics instead of
 734            // the generic vectorized code. This has two benefits: (a) we can take advantage of specific instructions
 735            // like pmovmskb which we know are optimized, and (b) we can avoid downclocking the processor while
 736            // this method is running.
 737
 738#if NET
 16739            if (!Vector512.IsHardwareAccelerated &&
 16740                !Vector256.IsHardwareAccelerated &&
 16741                (Sse2.IsSupported || AdvSimd.IsSupported))
 742            {
 0743                return GetIndexOfFirstNonAsciiChar_Intrinsified(pBuffer, bufferLength);
 744            }
 745            else
 746#endif
 747            {
 748                // Handles Vector512, Vector256, Vector128, and scalar.
 16749                return GetIndexOfFirstNonAsciiChar_Vector(pBuffer, bufferLength);
 750            }
 751        }
 752
 753        private static unsafe nuint GetIndexOfFirstNonAsciiChar_Vector(char* pBuffer, nuint bufferLength /* in chars */)
 754        {
 755            // Squirrel away the original buffer reference.This method works by determining the exact
 756            // char reference where non-ASCII data begins, so we need this base value to perform the
 757            // final subtraction at the end of the method to get the index into the original buffer.
 16758            char* pOriginalBuffer = pBuffer;
 759
 760#if SYSTEM_PRIVATE_CORELIB
 16761            Debug.Assert(bufferLength <= nuint.MaxValue / sizeof(char));
 762#endif
 763
 764#if NET
 765            // Before we drain off char-by-char, try a generic vectorized loop.
 766            // Only run the loop if we have at least two vectors we can pull out.
 16767            if (Vector512.IsHardwareAccelerated && bufferLength >= 2 * (uint)Vector512<ushort>.Count)
 768            {
 769                const uint SizeOfVector512InChars = Vector512.Size / sizeof(ushort);
 770
 0771                if (!VectorContainsNonAsciiChar(Vector512.Load((ushort*)pBuffer)))
 772                {
 773                    // The first several elements of the input buffer were ASCII. Bump up the pointer to the
 774                    // next aligned boundary, then perform aligned reads from here on out until we find non-ASCII
 775                    // data or we approach the end of the buffer. It's possible we'll reread data; this is ok.
 776
 0777                    char* pFinalVectorReadPos = pBuffer + bufferLength - SizeOfVector512InChars;
 0778                    pBuffer = (char*)(((nuint)pBuffer + Vector512.Size) & ~(nuint)(Vector512.Size - 1));
 779
 780#if DEBUG
 0781                    long numCharsRead = pBuffer - pOriginalBuffer;
 0782                    Debug.Assert(0 < numCharsRead && numCharsRead <= SizeOfVector512InChars, "We should've made forward 
 0783                    Debug.Assert((nuint)numCharsRead <= bufferLength, "We shouldn't have read past the end of the input 
 784#endif
 785
 0786                    Debug.Assert(pBuffer <= pFinalVectorReadPos, "Should be able to read at least one vector.");
 787
 788                    do
 789                    {
 0790                        Debug.Assert((nuint)pBuffer % Vector512.Size == 0, "Vector read should be aligned.");
 0791                        if (VectorContainsNonAsciiChar(Vector512.LoadAligned((ushort*)pBuffer)))
 792                        {
 793                            break; // found non-ASCII data
 794                        }
 0795                        pBuffer += SizeOfVector512InChars;
 0796                    } while (pBuffer <= pFinalVectorReadPos);
 797
 798                    // Adjust the remaining buffer length for the number of elements we just consumed.
 799
 0800                    bufferLength -= ((nuint)pBuffer - (nuint)pOriginalBuffer) / sizeof(char);
 801                }
 802            }
 16803            else if (Vector256.IsHardwareAccelerated && bufferLength >= 2 * (uint)Vector256<ushort>.Count)
 804            {
 805                const uint SizeOfVector256InChars = Vector256.Size / sizeof(ushort);
 806
 2807                if (!VectorContainsNonAsciiChar(Vector256.Load((ushort*)pBuffer)))
 808                {
 809                    // The first several elements of the input buffer were ASCII. Bump up the pointer to the
 810                    // next aligned boundary, then perform aligned reads from here on out until we find non-ASCII
 811                    // data or we approach the end of the buffer. It's possible we'll reread data; this is ok.
 812
 2813                    char* pFinalVectorReadPos = pBuffer + bufferLength - SizeOfVector256InChars;
 2814                    pBuffer = (char*)(((nuint)pBuffer + Vector256.Size) & ~(nuint)(Vector256.Size - 1));
 815
 816#if DEBUG
 2817                    long numCharsRead = pBuffer - pOriginalBuffer;
 2818                    Debug.Assert(0 < numCharsRead && numCharsRead <= SizeOfVector256InChars, "We should've made forward 
 2819                    Debug.Assert((nuint)numCharsRead <= bufferLength, "We shouldn't have read past the end of the input 
 820#endif
 821
 2822                    Debug.Assert(pBuffer <= pFinalVectorReadPos, "Should be able to read at least one vector.");
 823
 824                    do
 825                    {
 3826                        Debug.Assert((nuint)pBuffer % Vector256.Size == 0, "Vector read should be aligned.");
 3827                        if (VectorContainsNonAsciiChar(Vector256.LoadAligned((ushort*)pBuffer)))
 828                        {
 829                            break; // found non-ASCII data
 830                        }
 3831                        pBuffer += SizeOfVector256InChars;
 3832                    } while (pBuffer <= pFinalVectorReadPos);
 833
 834                    // Adjust the remaining buffer length for the number of elements we just consumed.
 835
 2836                    bufferLength -= ((nuint)pBuffer - (nuint)pOriginalBuffer) / sizeof(char);
 837                }
 838            }
 14839            else if (Vector128.IsHardwareAccelerated && bufferLength >= 2 * (uint)Vector128<ushort>.Count)
 840            {
 841                const uint SizeOfVector128InChars = Vector128.Size / sizeof(ushort); // JIT will make this a const
 842
 2843                if (!VectorContainsNonAsciiChar(Vector128.Load((ushort*)pBuffer)))
 844                {
 845                    // The first several elements of the input buffer were ASCII. Bump up the pointer to the
 846                    // next aligned boundary, then perform aligned reads from here on out until we find non-ASCII
 847                    // data or we approach the end of the buffer. It's possible we'll reread data; this is ok.
 2848                    char* pFinalVectorReadPos = pBuffer + bufferLength - SizeOfVector128InChars;
 2849                    pBuffer = (char*)(((nuint)pBuffer + Vector128.Size) & ~(nuint)(Vector128.Size - 1));
 850
 851#if DEBUG
 2852                    long numCharsRead = pBuffer - pOriginalBuffer;
 2853                    Debug.Assert(0 < numCharsRead && numCharsRead <= SizeOfVector128InChars, "We should've made forward 
 2854                    Debug.Assert((nuint)numCharsRead <= bufferLength, "We shouldn't have read past the end of the input 
 855#endif
 856
 2857                    Debug.Assert(pBuffer <= pFinalVectorReadPos, "Should be able to read at least one vector.");
 858
 859                    do
 860                    {
 5861                        Debug.Assert((nuint)pBuffer % Vector128.Size == 0, "Vector read should be aligned.");
 5862                        if (VectorContainsNonAsciiChar(Vector128.LoadAligned((ushort*)pBuffer)))
 863                        {
 864                            break; // found non-ASCII data
 865                        }
 5866                        pBuffer += SizeOfVector128InChars;
 5867                    } while (pBuffer <= pFinalVectorReadPos);
 868
 869                    // Adjust the remaining buffer length for the number of elements we just consumed.
 870
 2871                    bufferLength -= ((nuint)pBuffer - (nuint)pOriginalBuffer) / sizeof(char);
 872                }
 873            }
 874#endif
 875
 876            // At this point, the buffer length wasn't enough to perform a vectorized search, or we did perform
 877            // a vectorized search and encountered non-ASCII data. In either case go down a non-vectorized code
 878            // path to drain any remaining ASCII chars.
 879            //
 880            // We're going to perform unaligned reads, so prefer 32-bit reads instead of 64-bit reads.
 881            // This also allows us to perform more optimized bit twiddling tricks to count the number of ASCII chars.
 882
 883            uint currentUInt32;
 884
 885            // Try reading 64 bits at a time in a loop.
 886
 56887            for (; bufferLength >= 4; bufferLength -= 4) // 64 bits = 4 * 16-bit chars
 888            {
 20889                currentUInt32 = Unsafe.ReadUnaligned<uint>(pBuffer);
 20890                uint nextUInt32 = Unsafe.ReadUnaligned<uint>(pBuffer + 4 / sizeof(char));
 891
 20892                if (!AllCharsInUInt32AreAscii(currentUInt32 | nextUInt32))
 893                {
 894                    // One of these two values contains non-ASCII chars.
 895                    // Figure out which one it is, then put it in 'current' so that we can drain the ASCII chars.
 896
 0897                    if (AllCharsInUInt32AreAscii(currentUInt32))
 898                    {
 0899                        currentUInt32 = nextUInt32;
 0900                        pBuffer += 2;
 901                    }
 902
 0903                    goto FoundNonAsciiData;
 904                }
 905
 20906                pBuffer += 4; // consumed 4 ASCII chars
 907            }
 908
 909            // From this point forward we don't need to keep track of the remaining buffer length.
 910            // Try reading 32 bits.
 911
 16912            if ((bufferLength & 2) != 0) // 32 bits = 2 * 16-bit chars
 913            {
 6914                currentUInt32 = Unsafe.ReadUnaligned<uint>(pBuffer);
 6915                if (!AllCharsInUInt32AreAscii(currentUInt32))
 916                {
 917                    goto FoundNonAsciiData;
 918                }
 919
 6920                pBuffer += 2;
 921            }
 922
 923            // Try reading 16 bits.
 924            // No need to try an 8-bit read after this since we're working with chars.
 925
 16926            if ((bufferLength & 1) != 0)
 927            {
 928                // If the buffer contains non-ASCII data, the comparison below will fail, and
 929                // we'll end up not incrementing the buffer reference.
 930
 7931                if (*pBuffer <= 0x007F)
 932                {
 7933                    pBuffer++;
 934                }
 935            }
 936
 937        Finish:
 938
 16939            nuint totalNumBytesRead = (nuint)pBuffer - (nuint)pOriginalBuffer;
 16940            Debug.Assert(totalNumBytesRead % sizeof(char) == 0, "Total number of bytes read should be even since we're w
 16941            return totalNumBytesRead / sizeof(char); // convert byte count -> char count before returning
 942
 943        FoundNonAsciiData:
 944
 0945            Debug.Assert(!AllCharsInUInt32AreAscii(currentUInt32), "Shouldn't have reached this point if we have an all-
 946
 947            // We don't bother looking at the second char - only the first char.
 948
 0949            if (FirstCharInUInt32IsAscii(currentUInt32))
 950            {
 0951                pBuffer++;
 952            }
 953
 0954            goto Finish;
 955        }
 956
 957#if NET
 958        private static unsafe nuint GetIndexOfFirstNonAsciiChar_Intrinsified(char* pBuffer, nuint bufferLength /* in cha
 959        {
 960            // This method contains logic optimized using vector instructions for both x64 and Arm64.
 961            // Much of the logic in this method will be elided by JIT once we determine which specific ISAs we support.
 962
 963            // Quick check for empty inputs.
 964
 965            if (bufferLength == 0)
 966            {
 0967                return 0;
 968            }
 969
 970            // JIT turns the below into constants
 971
 0972            uint SizeOfVector128InChars = Vector128.Size / sizeof(char);
 973
 0974            Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported, "Should've been checked by caller.");
 0975            Debug.Assert(BitConverter.IsLittleEndian, "This SSE2/Arm64 assumes little-endian.");
 976
 977            Vector128<ushort> firstVector, secondVector;
 978            uint currentMask;
 0979            char* pOriginalBuffer = pBuffer;
 980
 0981            if (bufferLength < SizeOfVector128InChars)
 982            {
 983                goto InputBufferLessThanOneVectorInLength; // can't vectorize; drain primitives instead
 984            }
 985
 986            // This method is written such that control generally flows top-to-bottom, avoiding
 987            // jumps as much as possible in the optimistic case of "all ASCII". If we see non-ASCII
 988            // data, we jump out of the hot paths to targets at the end of the method.
 989
 990#if SYSTEM_PRIVATE_CORELIB
 0991            Debug.Assert(bufferLength <= nuint.MaxValue / sizeof(char));
 992#endif
 993
 994            // Read the first vector unaligned.
 995
 0996            firstVector = Vector128.LoadUnsafe(ref *(ushort*)pBuffer);
 0997            if (VectorContainsNonAsciiChar(firstVector))
 998            {
 999                goto FoundNonAsciiDataInFirstVector;
 1000            }
 1001
 1002            // If we have less than 32 bytes to process, just go straight to the final unaligned
 1003            // read. There's no need to mess with the loop logic in the middle of this method.
 1004
 1005            // Adjust the remaining length to account for what we just read.
 1006            // For the remainder of this code path, bufferLength will be in bytes, not chars.
 1007
 01008            bufferLength <<= 1; // chars to bytes
 1009
 01010            if (bufferLength < 2 * Vector128.Size)
 1011            {
 1012                goto IncrementCurrentOffsetBeforeFinalUnalignedVectorRead;
 1013            }
 1014
 1015            // Now adjust the read pointer so that future reads are aligned.
 1016
 01017            pBuffer = (char*)(((nuint)pBuffer + Vector128.Size) & ~(nuint)(Vector128.Size - 1));
 1018
 1019#if DEBUG
 01020            long numCharsRead = pBuffer - pOriginalBuffer;
 01021            Debug.Assert(0 < numCharsRead && numCharsRead <= SizeOfVector128InChars, "We should've made forward progress
 01022            Debug.Assert((nuint)numCharsRead <= bufferLength, "We shouldn't have read past the end of the input buffer."
 1023#endif
 1024
 1025            // Adjust remaining buffer length.
 1026
 01027            nuint numBytesRead = ((nuint)pBuffer - (nuint)pOriginalBuffer);
 01028            bufferLength -= numBytesRead;
 1029
 1030            // The buffer is now properly aligned.
 1031            // Read 2 vectors at a time if possible.
 01032            if (bufferLength >= 2 * Vector128.Size)
 1033            {
 01034                char* pFinalVectorReadPos = (char*)((nuint)pBuffer + bufferLength - 2 * Vector128.Size);
 1035
 1036                // After this point, we no longer need to update the bufferLength value.
 1037                do
 1038                {
 1039
 01040                    firstVector = Vector128.LoadUnsafe(ref *(ushort*)pBuffer);
 01041                    secondVector = Vector128.LoadUnsafe(ref *(ushort*)pBuffer, SizeOfVector128InChars);
 01042                    Vector128<ushort> combinedVector = firstVector | secondVector;
 1043
 01044                    if (VectorContainsNonAsciiChar(combinedVector))
 1045                    {
 1046                        goto FoundNonAsciiDataInFirstOrSecondVector;
 1047                    }
 1048
 01049                    pBuffer += 2 * SizeOfVector128InChars;
 01050                } while (pBuffer <= pFinalVectorReadPos);
 1051            }
 1052
 1053            // We have somewhere between 0 and (2 * vector length) - 1 bytes remaining to read from.
 1054            // Since the above loop doesn't update bufferLength, we can't rely on its absolute value.
 1055            // But we _can_ rely on it to tell us how much remaining data must be drained by looking
 1056            // at what bits of it are set. This works because had we updated it within the loop above,
 1057            // we would've been adding 2 * SizeOfVector128 on each iteration, but we only care about
 1058            // bits which are less significant than those that the addition would've acted on.
 1059
 1060            // If there is fewer than one vector length remaining, skip the next aligned read.
 1061            // Remember, at this point bufferLength is measured in bytes, not chars.
 1062
 01063            if ((bufferLength & Vector128.Size) == 0)
 1064            {
 1065                goto DoFinalUnalignedVectorRead;
 1066            }
 1067
 1068            // At least one full vector's worth of data remains, so we can safely read it.
 1069            // Remember, at this point pBuffer is still aligned.
 1070
 01071            firstVector = Vector128.LoadUnsafe(ref *(ushort*)pBuffer);
 01072            if (VectorContainsNonAsciiChar(firstVector))
 1073            {
 1074                goto FoundNonAsciiDataInFirstVector;
 1075            }
 1076
 1077        IncrementCurrentOffsetBeforeFinalUnalignedVectorRead:
 1078
 01079            pBuffer += SizeOfVector128InChars;
 1080
 1081        DoFinalUnalignedVectorRead:
 1082
 01083            if (((byte)bufferLength & (Vector128.Size - 1)) != 0)
 1084            {
 1085                // Perform an unaligned read of the last vector.
 1086                // We need to adjust the pointer because we're re-reading data.
 1087
 01088                pBuffer = (char*)((byte*)pBuffer + (bufferLength & (Vector128.Size - 1)) - Vector128.Size);
 01089                firstVector = Vector128.LoadUnsafe(ref *(ushort*)pBuffer);
 01090                if (VectorContainsNonAsciiChar(firstVector))
 1091                {
 1092                    goto FoundNonAsciiDataInFirstVector;
 1093                }
 1094
 01095                pBuffer += SizeOfVector128InChars;
 1096            }
 1097
 1098        Finish:
 1099
 01100            Debug.Assert(((nuint)pBuffer - (nuint)pOriginalBuffer) % 2 == 0, "Shouldn't have incremented any pointer by 
 01101            return ((nuint)pBuffer - (nuint)pOriginalBuffer) / sizeof(char); // and we're done! (remember to adjust for 
 1102
 1103        FoundNonAsciiDataInFirstOrSecondVector:
 1104
 1105            // We don't know if the first or the second vector contains non-ASCII data. Check the first
 1106            // vector, and if that's all-ASCII then the second vector must be the culprit. Either way
 1107            // we'll make sure the first vector local is the one that contains the non-ASCII data.
 1108
 01109            if (VectorContainsNonAsciiChar(firstVector))
 1110            {
 1111                goto FoundNonAsciiDataInFirstVector;
 1112            }
 1113
 1114            // Wasn't the first vector; must be the second.
 1115
 01116            pBuffer += SizeOfVector128InChars;
 01117            firstVector = secondVector;
 1118
 1119        FoundNonAsciiDataInFirstVector:
 1120
 01121            if (Sse2.IsSupported)
 1122            {
 1123                // The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element
 1124                // has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order
 1125                // to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored.
 01126                Vector128<ushort> asciiMaskForAddSaturate = Vector128.Create((ushort)0x7F80);
 1127                const uint NonAsciiDataSeenMask = 0b_1010_1010_1010_1010; // used for determining whether 'currentMask' 
 1128
 01129                currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForAddSaturate).AsByte());
 01130                currentMask &= NonAsciiDataSeenMask;
 1131
 1132                // Now, the mask contains - from the LSB - a 0b00 pair for each ASCII char we saw, and a 0b10 pair for e
 1133                //
 1134                // (Keep endianness in mind in the below examples.)
 1135                // A non-ASCII char followed by two ASCII chars is 0b..._00_00_10. (tzcnt = 1)
 1136                // An ASCII char followed by two non-ASCII chars is 0b..._10_10_00. (tzcnt = 3)
 1137                // Two ASCII chars followed by a non-ASCII char is 0b..._10_00_00. (tzcnt = 5)
 1138                //
 1139                // This means tzcnt = 2 * numLeadingAsciiChars + 1. We can conveniently take advantage of the fact
 1140                // that the 2x multiplier already matches the char* stride length, then just subtract 1 at the end to
 1141                // compute the correct final ending pointer value.
 1142
 01143                Debug.Assert(currentMask != 0, "Shouldn't be here unless we see non-ASCII data.");
 01144                pBuffer = (char*)((byte*)pBuffer + (uint)BitOperations.TrailingZeroCount(currentMask) - 1);
 1145            }
 1146            else if (AdvSimd.Arm64.IsSupported)
 1147            {
 1148                // The following operation sets all the bits in a WORD to 1 where a non-ASCII char is found (otherwise t
 1149                // in the vector. Then narrow each char to a byte by taking its top byte. Now the bottom-half (64-bits)
 1150                // of the vector contains 0xFFFF for non-ASCII and 0x0000 for ASCII char. We then find the index of the
 1151                // first non-ASCII char by counting number of trailing zeros representing ASCII chars before it.
 1152
 1153                Vector128<ushort> largestAsciiValue = Vector128.Create((ushort)0x007F);
 1154                Vector128<byte> compareResult = AdvSimd.CompareGreaterThan(firstVector, largestAsciiValue).AsByte();
 1155                ulong asciiCompareMask = AdvSimd.Arm64.UnzipOdd(compareResult, compareResult).AsUInt64().ToScalar();
 1156                // Compare mask now contains 8 bits for each 16-bit char. Divide it by 8 to get to the first non-ASCII b
 1157                pBuffer += BitOperations.TrailingZeroCount(asciiCompareMask) >> 3;
 1158            }
 1159            else
 1160            {
 01161                throw new PlatformNotSupportedException();
 1162            }
 1163            goto Finish;
 1164
 1165        FoundNonAsciiDataInCurrentDWord:
 1166
 1167            uint currentDWord;
 01168            Debug.Assert(!AllCharsInUInt32AreAscii(currentDWord), "Shouldn't be here unless we see non-ASCII data.");
 1169
 01170            if (FirstCharInUInt32IsAscii(currentDWord))
 1171            {
 01172                pBuffer++; // skip past the ASCII char
 1173            }
 1174
 01175            goto Finish;
 1176
 1177        InputBufferLessThanOneVectorInLength:
 1178
 1179            // These code paths get hit if the original input length was less than one vector in size.
 1180            // We can't perform vectorized reads at this point, so we'll fall back to reading primitives
 1181            // directly. Note that all of these reads are unaligned.
 1182
 1183            // Reminder: If this code path is hit, bufferLength is still a char count, not a byte count.
 1184            // We skipped the code path that multiplied the count by sizeof(char).
 1185
 01186            Debug.Assert(bufferLength < SizeOfVector128InChars);
 1187
 1188            // QWORD drain
 1189
 01190            if ((bufferLength & 4) != 0)
 1191            {
 1192                if (UIntPtr.Size == sizeof(ulong))
 1193                {
 1194                    // If we can use 64-bit tzcnt to count the number of leading ASCII chars, prefer it.
 1195
 01196                    ulong candidateUInt64 = Unsafe.ReadUnaligned<ulong>(pBuffer);
 01197                    if (!AllCharsInUInt64AreAscii(candidateUInt64))
 1198                    {
 1199                        // Clear the low 7 bits (the ASCII bits) of each char, then tzcnt.
 1200                        // Remember to divide by 8 at the end to convert bit count to byte count,
 1201                        // then the & ~1 at the end to treat a match in the high byte of
 1202                        // any char the same as a match in the low byte of that same char.
 1203
 01204                        candidateUInt64 &= 0xFF80FF80_FF80FF80ul;
 01205                        pBuffer = (char*)((byte*)pBuffer + ((nuint)(BitOperations.TrailingZeroCount(candidateUInt64) >> 
 01206                        goto Finish;
 1207                    }
 1208                }
 1209                else
 1210                {
 1211                    // If we can't use 64-bit tzcnt, no worries. We'll just do 2x 32-bit reads instead.
 1212
 1213                    currentDWord = Unsafe.ReadUnaligned<uint>(pBuffer);
 1214                    uint nextDWord = Unsafe.ReadUnaligned<uint>(pBuffer + 4 / sizeof(char));
 1215
 1216                    if (!AllCharsInUInt32AreAscii(currentDWord | nextDWord))
 1217                    {
 1218                        // At least one of the values wasn't all-ASCII.
 1219                        // We need to figure out which one it was and stick it in the currentMask local.
 1220
 1221                        if (AllCharsInUInt32AreAscii(currentDWord))
 1222                        {
 1223                            currentDWord = nextDWord; // this one is the culprit
 1224                            pBuffer += 4 / sizeof(char);
 1225                        }
 1226
 1227                        goto FoundNonAsciiDataInCurrentDWord;
 1228                    }
 1229                }
 1230
 01231                pBuffer += 4; // successfully consumed 4 ASCII chars
 1232            }
 1233
 1234            // DWORD drain
 1235
 01236            if ((bufferLength & 2) != 0)
 1237            {
 01238                currentDWord = Unsafe.ReadUnaligned<uint>(pBuffer);
 1239
 01240                if (!AllCharsInUInt32AreAscii(currentDWord))
 1241                {
 1242                    goto FoundNonAsciiDataInCurrentDWord;
 1243                }
 1244
 01245                pBuffer += 2; // successfully consumed 2 ASCII chars
 1246            }
 1247
 1248            // WORD drain
 1249            // This is the final drain; there's no need for a BYTE drain since our elemental type is 16-bit char.
 1250
 01251            if ((bufferLength & 1) != 0)
 1252            {
 01253                if (*pBuffer <= 0x007F)
 1254                {
 01255                    pBuffer++; // successfully consumed a single char
 1256                }
 1257            }
 1258
 01259            goto Finish;
 1260        }
 1261#endif
 1262
 1263        /// <summary>
 1264        /// Given a QWORD which represents a buffer of 4 ASCII chars in machine-endian order,
 1265        /// narrows each WORD to a BYTE, then writes the 4-byte result to the output buffer
 1266        /// also in machine-endian order.
 1267        /// </summary>
 1268        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1269        private static void NarrowFourUtf16CharsToAsciiAndWriteToBuffer(ref byte outputBuffer, ulong value)
 1270        {
 1271            Debug.Assert(AllCharsInUInt64AreAscii(value));
 1272
 1273#if NET
 143821274            if (Sse2.X64.IsSupported)
 1275            {
 1276                // Narrows a vector of words [ w0 w1 w2 w3 ] to a vector of bytes
 1277                // [ b0 b1 b2 b3 b0 b1 b2 b3 ], then writes 4 bytes (32 bits) to the destination.
 1278
 143821279                Vector128<short> vecWide = Sse2.X64.ConvertScalarToVector128UInt64(value).AsInt16();
 143821280                Vector128<uint> vecNarrow = Sse2.PackUnsignedSaturate(vecWide, vecWide).AsUInt32();
 143821281                Unsafe.WriteUnaligned(ref outputBuffer, Sse2.ConvertToUInt32(vecNarrow));
 1282            }
 1283            else if (AdvSimd.IsSupported)
 1284            {
 1285                // Narrows a vector of words [ w0 w1 w2 w3 ] to a vector of bytes
 1286                // [ b0 b1 b2 b3 * * * * ], then writes 4 bytes (32 bits) to the destination.
 1287
 1288                Vector128<short> vecWide = Vector128.CreateScalarUnsafe(value).AsInt16();
 1289                Vector64<byte> lower = AdvSimd.ExtractNarrowingSaturateUnsignedLower(vecWide);
 1290                Unsafe.WriteUnaligned(ref outputBuffer, lower.AsUInt32().ToScalar());
 1291            }
 1292            else
 1293#endif
 1294            {
 01295                if (BitConverter.IsLittleEndian)
 1296                {
 01297                    outputBuffer = (byte)value;
 01298                    value >>= 16;
 01299                    Unsafe.Add(ref outputBuffer, 1) = (byte)value;
 01300                    value >>= 16;
 01301                    Unsafe.Add(ref outputBuffer, 2) = (byte)value;
 01302                    value >>= 16;
 01303                    Unsafe.Add(ref outputBuffer, 3) = (byte)value;
 1304                }
 1305                else
 1306                {
 1307                    Unsafe.Add(ref outputBuffer, 3) = (byte)value;
 1308                    value >>= 16;
 1309                    Unsafe.Add(ref outputBuffer, 2) = (byte)value;
 1310                    value >>= 16;
 1311                    Unsafe.Add(ref outputBuffer, 1) = (byte)value;
 1312                    value >>= 16;
 1313                    outputBuffer = (byte)value;
 1314                }
 1315            }
 1316        }
 1317
 1318        /// <summary>
 1319        /// Given a DWORD which represents a buffer of 2 ASCII chars in machine-endian order,
 1320        /// narrows each WORD to a BYTE, then writes the 2-byte result to the output buffer also in
 1321        /// machine-endian order.
 1322        /// </summary>
 1323        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1324        private static void NarrowTwoUtf16CharsToAsciiAndWriteToBuffer(ref byte outputBuffer, uint value)
 1325        {
 1326            Debug.Assert(AllCharsInUInt32AreAscii(value));
 1327
 24171328            if (BitConverter.IsLittleEndian)
 1329            {
 24171330                outputBuffer = (byte)value;
 24171331                Unsafe.Add(ref outputBuffer, 1) = (byte)(value >> 16);
 1332            }
 1333            else
 1334            {
 1335                Unsafe.Add(ref outputBuffer, 1) = (byte)value;
 1336                outputBuffer = (byte)(value >> 16);
 1337            }
 1338        }
 1339
 1340        /// <summary>
 1341        /// Copies as many ASCII characters (U+0000..U+007F) as possible from <paramref name="pUtf16Buffer"/>
 1342        /// to <paramref name="pAsciiBuffer"/>, stopping when the first non-ASCII character is encountered
 1343        /// or once <paramref name="elementCount"/> elements have been converted. Returns the total number
 1344        /// of elements that were able to be converted.
 1345        /// </summary>
 1346        internal static unsafe nuint NarrowUtf16ToAscii(char* pUtf16Buffer, byte* pAsciiBuffer, nuint elementCount)
 1347        {
 1348            nuint currentOffset = 0;
 1349
 150161350            uint utf16Data32BitsHigh = 0, utf16Data32BitsLow = 0;
 75081351            ulong utf16Data64Bits = 0;
 1352
 1353#if NET
 75081354            if (BitConverter.IsLittleEndian && Vector128.IsHardwareAccelerated && elementCount >= 2 * (uint)Vector128<by
 1355            {
 1356                // Since there's overhead to setting up the vectorized code path, we only want to
 1357                // call into it after a quick probe to ensure the next immediate characters really are ASCII.
 1358                // If we see non-ASCII data, we'll jump immediately to the draining logic at the end of the method.
 1359
 1360                if (IntPtr.Size >= 8)
 1361                {
 38621362                    utf16Data64Bits = Unsafe.ReadUnaligned<ulong>(pUtf16Buffer);
 38621363                    if (!AllCharsInUInt64AreAscii(utf16Data64Bits))
 1364                    {
 14351365                        goto FoundNonAsciiDataIn64BitRead;
 1366                    }
 1367                }
 1368                else
 1369                {
 1370                    utf16Data32BitsHigh = Unsafe.ReadUnaligned<uint>(pUtf16Buffer);
 1371                    utf16Data32BitsLow = Unsafe.ReadUnaligned<uint>(pUtf16Buffer + 4 / sizeof(char));
 1372                    if (!AllCharsInUInt32AreAscii(utf16Data32BitsHigh | utf16Data32BitsLow))
 1373                    {
 1374                        goto FoundNonAsciiDataIn64BitRead;
 1375                    }
 1376                }
 24271377                if (Vector512.IsHardwareAccelerated && elementCount >= 2 * (uint)Vector512<byte>.Count)
 1378                {
 8361379                    currentOffset = NarrowUtf16ToAscii_Intrinsified_512(pUtf16Buffer, pAsciiBuffer, elementCount);
 1380                }
 15911381                else if (Vector256.IsHardwareAccelerated && elementCount >= 2 * (uint)Vector256<byte>.Count)
 1382                {
 5661383                    currentOffset = NarrowUtf16ToAscii_Intrinsified_256(pUtf16Buffer, pAsciiBuffer, elementCount);
 1384                }
 1385                else
 1386                {
 10251387                    currentOffset = NarrowUtf16ToAscii_Intrinsified(pUtf16Buffer, pAsciiBuffer, elementCount);
 1388                }
 1389            }
 1390#endif
 1391
 60731392            Debug.Assert(currentOffset <= elementCount);
 60731393            nuint remainingElementCount = elementCount - currentOffset;
 1394
 1395            // Try to narrow 64 bits -> 32 bits at a time.
 1396            // We needn't update remainingElementCount after this point.
 1397
 60731398            if (remainingElementCount >= 4)
 1399            {
 53351400                nuint finalOffsetWhereCanLoop = currentOffset + remainingElementCount - 4;
 1401                do
 1402                {
 1403                    if (IntPtr.Size >= 8)
 1404                    {
 1405                        // Only perform QWORD reads on a 64-bit platform.
 161031406                        utf16Data64Bits = Unsafe.ReadUnaligned<ulong>(pUtf16Buffer + currentOffset);
 161031407                        if (!AllCharsInUInt64AreAscii(utf16Data64Bits))
 1408                        {
 1409                            goto FoundNonAsciiDataIn64BitRead;
 1410                        }
 1411
 143821412                        NarrowFourUtf16CharsToAsciiAndWriteToBuffer(ref pAsciiBuffer[currentOffset], utf16Data64Bits);
 1413                    }
 1414                    else
 1415                    {
 1416                        utf16Data32BitsHigh = Unsafe.ReadUnaligned<uint>(pUtf16Buffer + currentOffset);
 1417                        utf16Data32BitsLow = Unsafe.ReadUnaligned<uint>(pUtf16Buffer + currentOffset + 4 / sizeof(char))
 1418                        if (!AllCharsInUInt32AreAscii(utf16Data32BitsHigh | utf16Data32BitsLow))
 1419                        {
 1420                            goto FoundNonAsciiDataIn64BitRead;
 1421                        }
 1422
 1423                        NarrowTwoUtf16CharsToAsciiAndWriteToBuffer(ref pAsciiBuffer[currentOffset], utf16Data32BitsHigh)
 1424                        NarrowTwoUtf16CharsToAsciiAndWriteToBuffer(ref pAsciiBuffer[currentOffset + 2], utf16Data32BitsL
 1425                    }
 1426
 143821427                    currentOffset += 4;
 143821428                } while (currentOffset <= finalOffsetWhereCanLoop);
 1429            }
 1430
 1431            // Try to narrow 32 bits -> 16 bits.
 1432
 43521433            if (((uint)remainingElementCount & 2) != 0)
 1434            {
 21661435                utf16Data32BitsHigh = Unsafe.ReadUnaligned<uint>(pUtf16Buffer + currentOffset);
 21661436                if (!AllCharsInUInt32AreAscii(utf16Data32BitsHigh))
 1437                {
 1438                    goto FoundNonAsciiDataInHigh32Bits;
 1439                }
 1440
 19401441                NarrowTwoUtf16CharsToAsciiAndWriteToBuffer(ref pAsciiBuffer[currentOffset], utf16Data32BitsHigh);
 19401442                currentOffset += 2;
 1443            }
 1444
 1445            // Try to narrow 16 bits -> 8 bits.
 1446
 41261447            if (((uint)remainingElementCount & 1) != 0)
 1448            {
 19231449                utf16Data32BitsHigh = pUtf16Buffer[currentOffset];
 19231450                if (utf16Data32BitsHigh <= 0x007Fu)
 1451                {
 18761452                    pAsciiBuffer[currentOffset] = (byte)utf16Data32BitsHigh;
 18761453                    currentOffset++;
 1454                }
 1455            }
 1456
 1457        Finish:
 1458
 75081459            return currentOffset;
 1460
 1461        FoundNonAsciiDataIn64BitRead:
 1462
 1463            if (IntPtr.Size >= 8)
 1464            {
 1465                // Try checking the first 32 bits of the buffer for non-ASCII data.
 1466                // Regardless, we'll move the non-ASCII data into the utf16Data32BitsHigh local.
 1467
 31561468                if (BitConverter.IsLittleEndian)
 1469                {
 31561470                    utf16Data32BitsHigh = (uint)utf16Data64Bits;
 1471                }
 1472                else
 1473                {
 1474                    utf16Data32BitsHigh = (uint)(utf16Data64Bits >> 32);
 1475                }
 1476
 31561477                if (AllCharsInUInt32AreAscii(utf16Data32BitsHigh))
 1478                {
 4771479                    NarrowTwoUtf16CharsToAsciiAndWriteToBuffer(ref pAsciiBuffer[currentOffset], utf16Data32BitsHigh);
 1480
 4771481                    if (BitConverter.IsLittleEndian)
 1482                    {
 4771483                        utf16Data32BitsHigh = (uint)(utf16Data64Bits >> 32);
 1484                    }
 1485                    else
 1486                    {
 1487                        utf16Data32BitsHigh = (uint)utf16Data64Bits;
 1488                    }
 1489
 4771490                    currentOffset += 2;
 1491                }
 1492            }
 1493            else
 1494            {
 1495                // Need to determine if the high or the low 32-bit value contained non-ASCII data.
 1496                // Regardless, we'll move the non-ASCII data into the utf16Data32BitsHigh local.
 1497
 1498                if (AllCharsInUInt32AreAscii(utf16Data32BitsHigh))
 1499                {
 1500                    NarrowTwoUtf16CharsToAsciiAndWriteToBuffer(ref pAsciiBuffer[currentOffset], utf16Data32BitsHigh);
 1501                    utf16Data32BitsHigh = utf16Data32BitsLow;
 1502                    currentOffset += 2;
 1503                }
 1504            }
 1505
 1506        FoundNonAsciiDataInHigh32Bits:
 1507
 33821508            Debug.Assert(!AllCharsInUInt32AreAscii(utf16Data32BitsHigh), "Shouldn't have reached this point if we have a
 1509
 1510            // There's at most one char that needs to be drained.
 1511
 33821512            if (FirstCharInUInt32IsAscii(utf16Data32BitsHigh))
 1513            {
 6851514                if (!BitConverter.IsLittleEndian)
 1515                {
 1516                    utf16Data32BitsHigh >>= 16; // move high char down to low char
 1517                }
 1518
 6851519                pAsciiBuffer[currentOffset] = (byte)utf16Data32BitsHigh;
 6851520                currentOffset++;
 1521            }
 1522
 6851523            goto Finish;
 1524        }
 1525
 1526#if NET
 1527        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1528        private static bool VectorContainsNonAsciiChar(Vector128<byte> asciiVector)
 1529        {
 1530            // max ASCII character is 0b_0111_1111, so the most significant bit (0x80) tells whether it contains non asc
 1531
 1532            // For performance, prefer architecture specific implementation
 1533            if (Sse41.IsSupported)
 1534            {
 1459321535                return (asciiVector & Vector128.Create((byte)0x80)) != Vector128<byte>.Zero;
 1536            }
 1537            else if (AdvSimd.Arm64.IsSupported)
 1538            {
 1539                Vector128<byte> maxBytes = AdvSimd.Arm64.MaxPairwise(asciiVector, asciiVector);
 1540                return (maxBytes.AsUInt64().ToScalar() & 0x8080808080808080) != 0;
 1541            }
 1542            else
 1543            {
 01544                return asciiVector.ExtractMostSignificantBits() != 0;
 1545            }
 1546        }
 1547
 1548        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1549        internal static bool VectorContainsNonAsciiChar(Vector128<ushort> utf16Vector)
 1550        {
 1551            // For performance, prefer architecture specific implementation
 1552            if (Sse41.IsSupported)
 1553            {
 1554                const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
 32451555                Vector128<ushort> zeroIsAscii = utf16Vector & Vector128.Create(asciiMask);
 1556                // If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
 32451557                return zeroIsAscii != Vector128<ushort>.Zero;
 1558            }
 01559            else if (Sse2.IsSupported)
 1560            {
 01561                Vector128<ushort> asciiMaskForAddSaturate = Vector128.Create((ushort)0x7F80);
 1562                // The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element
 1563                // has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order
 1564                // to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored.
 01565                return (Sse2.MoveMask(Sse2.AddSaturate(utf16Vector, asciiMaskForAddSaturate).AsByte()) & 0b_1010_1010_10
 1566            }
 1567            else if (AdvSimd.Arm64.IsSupported)
 1568            {
 1569                // First we pick four chars, a larger one from all four pairs of adjecent chars in the vector.
 1570                // If any of those four chars has a non-ASCII bit set, we have seen non-ASCII data.
 1571                Vector128<ushort> maxChars = AdvSimd.Arm64.MaxPairwise(utf16Vector, utf16Vector);
 1572                return (maxChars.AsUInt64().ToScalar() & 0xFF80FF80FF80FF80) != 0;
 1573            }
 1574            else
 1575            {
 1576                const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
 01577                Vector128<ushort> zeroIsAscii = utf16Vector & Vector128.Create(asciiMask);
 1578                // If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
 01579                return zeroIsAscii != Vector128<ushort>.Zero;
 1580            }
 1581        }
 1582
 1583        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1584        internal static bool VectorContainsNonAsciiChar(Vector256<ushort> utf16Vector)
 1585        {
 1586            const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
 17901587            Vector256<ushort> zeroIsAscii = utf16Vector & Vector256.Create(asciiMask);
 1588            // If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
 17901589            return zeroIsAscii != Vector256<ushort>.Zero;
 1590        }
 1591
 1592        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1593        internal static bool VectorContainsNonAsciiChar(Vector512<ushort> utf16Vector)
 1594        {
 1595            const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
 40341596            Vector512<ushort> zeroIsAscii = utf16Vector & Vector512.Create(asciiMask);
 1597            // If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
 40341598            return zeroIsAscii != Vector512<ushort>.Zero;
 1599        }
 1600
 1601        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1602        private static bool VectorContainsNonAsciiChar<T>(Vector128<T> vector)
 1603            where T : unmanaged
 1604        {
 21605            Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));
 1606
 21607            return typeof(T) == typeof(byte)
 21608                ? VectorContainsNonAsciiChar(vector.AsByte())
 21609                : VectorContainsNonAsciiChar(vector.AsUInt16());
 1610        }
 1611
 1612        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1613        private static bool AllCharsInVectorAreAscii<T>(Vector128<T> vector)
 1614            where T : unmanaged
 1615        {
 1616            Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));
 1617
 1618            // This is a copy of VectorContainsNonAsciiChar with an inverted condition.
 01619            if (typeof(T) == typeof(byte))
 1620            {
 01621                return
 01622                    Sse41.IsSupported ? (vector.AsByte() & Vector128.Create((byte)0x80)) == Vector128<byte>.Zero :
 01623                    AdvSimd.Arm64.IsSupported ? AllBytesInUInt64AreAscii(AdvSimd.Arm64.MaxPairwise(vector.AsByte(), vect
 01624                    vector.AsByte().ExtractMostSignificantBits() == 0;
 1625            }
 1626            else
 1627            {
 1628                return
 1629                    AdvSimd.Arm64.IsSupported ? AllCharsInUInt64AreAscii(AdvSimd.Arm64.MaxPairwise(vector.AsUInt16(), ve
 1630                    (vector.AsUInt16() & Vector128.Create((ushort)0xFF80)) == Vector128<ushort>.Zero;
 1631            }
 1632        }
 1633
 1634        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1635        [CompExactlyDependsOn(typeof(Avx))]
 1636        [CompHasFallback]
 1637        private static bool AllCharsInVectorAreAscii<T>(Vector256<T> vector)
 1638            where T : unmanaged
 1639        {
 01640            Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));
 1641
 01642            if (typeof(T) == typeof(byte))
 1643            {
 01644                return
 01645                    Avx.IsSupported ? (vector.AsByte() & Vector256.Create((byte)0x80)) == Vector256<byte>.Zero:
 01646                    vector.AsByte().ExtractMostSignificantBits() == 0;
 1647            }
 1648            else
 1649            {
 01650                return (vector.AsUInt16() & Vector256.Create((ushort)0xFF80)) == Vector256<ushort>.Zero;
 1651            }
 1652        }
 1653
 1654        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1655        private static bool AllCharsInVectorAreAscii<T>(Vector512<T> vector)
 1656            where T : unmanaged
 1657        {
 01658            Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));
 1659
 01660            if (typeof(T) == typeof(byte))
 1661            {
 01662                return vector.AsByte().ExtractMostSignificantBits() == 0;
 1663            }
 1664            else
 1665            {
 01666                return (vector.AsUInt16() & Vector512.Create((ushort)0xFF80)) == Vector512<ushort>.Zero;
 1667            }
 1668        }
 1669
 1670        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1671        internal static Vector128<byte> ExtractAsciiVector(Vector128<ushort> vectorFirst, Vector128<ushort> vectorSecond
 1672        {
 1673            // Narrows two vectors of words [ w7 w6 w5 w4 w3 w2 w1 w0 ] and [ w7' w6' w5' w4' w3' w2' w1' w0' ]
 1674            // to a vector of bytes [ b7 ... b0 b7' ... b0'].
 1675
 1676            // prefer architecture specific intrinsic as they don't perform additional AND like Vector128.Narrow does
 1677            if (Sse2.IsSupported)
 1678            {
 31541679                return Sse2.PackUnsignedSaturate(vectorFirst.AsInt16(), vectorSecond.AsInt16());
 1680            }
 1681            else if (AdvSimd.Arm64.IsSupported)
 1682            {
 1683                return AdvSimd.Arm64.UnzipEven(vectorFirst.AsByte(), vectorSecond.AsByte());
 1684            }
 1685            else if (PackedSimd.IsSupported)
 1686            {
 1687                return PackedSimd.ConvertNarrowingSaturateUnsigned(vectorFirst.AsInt16(), vectorSecond.AsInt16());
 1688            }
 1689            else
 1690            {
 01691                return Vector128.Narrow(vectorFirst, vectorSecond);
 1692            }
 1693        }
 1694
 1695        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1696        internal static Vector256<byte> ExtractAsciiVector(Vector256<ushort> vectorFirst, Vector256<ushort> vectorSecond
 1697        {
 17181698            return Avx2.IsSupported
 17181699                ? PackedSpanHelpers.FixUpPackedVector256Result(Avx2.PackUnsignedSaturate(vectorFirst.AsInt16(), vectorSe
 17181700                : Vector256.Narrow(vectorFirst, vectorSecond);
 1701        }
 1702
 1703        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1704        internal static Vector512<byte> ExtractAsciiVector(Vector512<ushort> vectorFirst, Vector512<ushort> vectorSecond
 1705        {
 39341706            return Avx512BW.IsSupported
 39341707                ? PackedSpanHelpers.FixUpPackedVector512Result(Avx512BW.PackUnsignedSaturate(vectorFirst.AsInt16(), vect
 39341708                : Vector512.Narrow(vectorFirst, vectorSecond);
 1709        }
 1710
 1711        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1712        private static unsafe nuint NarrowUtf16ToAscii_Intrinsified(char* pUtf16Buffer, byte* pAsciiBuffer, nuint elemen
 1713        {
 1714            // This method contains logic optimized using vector instructions for both x64 and Arm64.
 1715            // Much of the logic in this method will be elided by JIT once we determine which specific ISAs we support.
 1716
 1717            // JIT turns the below into constants
 1718
 10251719            uint SizeOfVector128 = (uint)Vector128<byte>.Count;
 10251720            nuint MaskOfAllBitsInVector128 = (nuint)(SizeOfVector128 - 1);
 1721
 1722            // This method is written such that control generally flows top-to-bottom, avoiding
 1723            // jumps as much as possible in the optimistic case of "all ASCII". If we see non-ASCII
 1724            // data, we jump out of the hot paths to targets at the end of the method.
 1725
 10251726            Debug.Assert(Vector128.IsHardwareAccelerated, "Vector128 is required.");
 10251727            Debug.Assert(BitConverter.IsLittleEndian, "This implementation assumes little-endian.");
 10251728            Debug.Assert(elementCount >= 2 * SizeOfVector128);
 1729
 1730            // First, perform an unaligned read of the first part of the input buffer.
 10251731            ref ushort utf16Buffer = ref *(ushort*)pUtf16Buffer;
 10251732            Vector128<ushort> utf16VectorFirst = Vector128.LoadUnsafe(ref utf16Buffer);
 1733
 1734            // If there's non-ASCII data in the first 8 elements of the vector, there's nothing we can do.
 10251735            if (VectorContainsNonAsciiChar(utf16VectorFirst))
 1736            {
 181737                return 0;
 1738            }
 1739
 1740            // Turn the 8 ASCII chars we just read into 8 ASCII bytes, then copy it to the destination.
 1741
 10071742            ref byte asciiBuffer = ref *pAsciiBuffer;
 10071743            Vector128<byte> asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 10071744            asciiVector.StoreLowerUnsafe(ref asciiBuffer, 0);
 10071745            nuint currentOffsetInElements = SizeOfVector128 / 2; // we processed 8 elements so far
 1746
 1747            // We're going to get the best performance when we have aligned writes, so we'll take the
 1748            // hit of potentially unaligned reads in order to hit this sweet spot.
 1749
 1750            // pAsciiBuffer points to the start of the destination buffer, immediately before where we wrote
 1751            // the 8 bytes previously. If the 0x08 bit is set at the pinned address, then the 8 bytes we wrote
 1752            // previously mean that the 0x08 bit is *not* set at address &pAsciiBuffer[SizeOfVector128 / 2]. In
 1753            // that case we can immediately back up to the previous aligned boundary and start the main loop.
 1754            // If the 0x08 bit is *not* set at the pinned address, then it means the 0x08 bit *is* set at
 1755            // address &pAsciiBuffer[SizeOfVector128 / 2], and we should perform one more 8-byte write to bump
 1756            // just past the next aligned boundary address.
 1757
 10071758            if (((uint)pAsciiBuffer & (SizeOfVector128 / 2)) == 0)
 1759            {
 1760                // We need to perform one more partial vector write before we can get the alignment we want.
 1761
 6361762                utf16VectorFirst = Vector128.LoadUnsafe(ref utf16Buffer, currentOffsetInElements);
 1763
 6361764                if (VectorContainsNonAsciiChar(utf16VectorFirst))
 1765                {
 1766                    goto Finish;
 1767                }
 1768
 1769                // Turn the 8 ASCII chars we just read into 8 ASCII bytes, then copy it to the destination.
 6211770                asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 6211771                asciiVector.StoreLowerUnsafe(ref asciiBuffer, currentOffsetInElements);
 1772            }
 1773
 1774            // Calculate how many elements we wrote in order to get pAsciiBuffer to its next alignment
 1775            // point, then use that as the base offset going forward.
 1776
 9921777            currentOffsetInElements = SizeOfVector128 - ((nuint)pAsciiBuffer & MaskOfAllBitsInVector128);
 1778
 9921779            Debug.Assert(0 < currentOffsetInElements && currentOffsetInElements <= SizeOfVector128, "We wrote at least 1
 9921780            Debug.Assert(currentOffsetInElements <= elementCount, "Shouldn't have overrun the destination buffer.");
 9921781            Debug.Assert(elementCount - currentOffsetInElements >= SizeOfVector128, "We should be able to run at least o
 1782
 9921783            nuint finalOffsetWhereCanRunLoop = elementCount - SizeOfVector128;
 1784            do
 1785            {
 1786                // In a loop, perform two unaligned reads, narrow to a single vector, then aligned write one vector.
 1787
 15391788                utf16VectorFirst = Vector128.LoadUnsafe(ref utf16Buffer, currentOffsetInElements);
 15391789                Vector128<ushort> utf16VectorSecond = Vector128.LoadUnsafe(ref utf16Buffer, currentOffsetInElements + Si
 15391790                Vector128<ushort> combinedVector = utf16VectorFirst | utf16VectorSecond;
 1791
 15391792                if (VectorContainsNonAsciiChar(combinedVector))
 1793                {
 1794                    goto FoundNonAsciiDataInLoop;
 1795                }
 1796
 1797                // Build up the ASCII vector and perform the store.
 1798
 15031799                Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % SizeOfVector128 == 0, "Write should be al
 15031800                asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorSecond);
 15031801                asciiVector.StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 1802
 15031803                currentOffsetInElements += SizeOfVector128;
 15031804            } while (currentOffsetInElements <= finalOffsetWhereCanRunLoop);
 1805
 1806        Finish:
 1807
 1808            // There might be some ASCII data left over. That's fine - we'll let our caller handle the final drain.
 10071809            return currentOffsetInElements;
 1810
 1811        FoundNonAsciiDataInLoop:
 1812
 1813            // Can we at least narrow the high vector?
 1814            // See comments in GetIndexOfFirstNonAsciiChar_Intrinsified for information about how this works.
 361815            if (VectorContainsNonAsciiChar(utf16VectorFirst))
 1816            {
 1817                goto Finish;
 1818            }
 1819
 1820            // First part was all ASCII, narrow and aligned write. Note we're only filling in the low half of the vector
 1821
 231822            Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % sizeof(ulong) == 0, "Destination should be ul
 231823            asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 231824            asciiVector.StoreLowerUnsafe(ref asciiBuffer, currentOffsetInElements);
 231825            currentOffsetInElements += SizeOfVector128 / 2;
 1826
 231827            goto Finish;
 1828        }
 1829
 1830        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1831        private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_256(char* pUtf16Buffer, byte* pAsciiBuffer, nuint el
 1832        {
 1833            // This method contains logic optimized using vector instructions for x64 only.
 1834            // Much of the logic in this method will be elided by JIT once we determine which specific ISAs we support.
 1835
 1836            // JIT turns the below into constants
 1837
 1838            const nuint MaskOfAllBitsInVector256 = (nuint)(Vector256.Size - 1);
 1839
 1840            // This method is written such that control generally flows top-to-bottom, avoiding
 1841            // jumps as much as possible in the optimistic case of "all ASCII". If we see non-ASCII
 1842            // data, we jump out of the hot paths to targets at the end of the method.
 1843
 5661844            Debug.Assert(Vector256.IsHardwareAccelerated, "Vector256 is required.");
 5661845            Debug.Assert(BitConverter.IsLittleEndian, "This implementation assumes little-endian.");
 5661846            Debug.Assert(elementCount >= 2 * Vector256.Size);
 1847
 1848            // First, perform an unaligned read of the first part of the input buffer.
 5661849            ref ushort utf16Buffer = ref *(ushort*)pUtf16Buffer;
 5661850            Vector256<ushort> utf16VectorFirst = Vector256.LoadUnsafe(ref utf16Buffer);
 1851
 1852            // If there's non-ASCII data in the first 16 elements of the vector, there's nothing we can do.
 5661853            if (VectorContainsNonAsciiChar(utf16VectorFirst))
 1854            {
 251855                return 0;
 1856            }
 1857
 1858            // Turn the 16 ASCII chars we just read into 16 ASCII bytes, then copy it to the destination.
 1859
 5411860            ref byte asciiBuffer = ref *pAsciiBuffer;
 5411861            Vector256<byte> asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 5411862            asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, 0);
 5411863            nuint currentOffsetInElements = Vector256.Size / 2; // we processed 16 elements so far
 1864
 1865            // We're going to get the best performance when we have aligned writes, so we'll take the
 1866            // hit of potentially unaligned reads in order to hit this sweet spot.
 1867
 1868            // pAsciiBuffer points to the start of the destination buffer, immediately before where we wrote
 1869            // the 16 bytes previously. If the 0x10 bit is set at the pinned address, then the 16 bytes we wrote
 1870            // previously mean that the 0x10 bit is *not* set at address &pAsciiBuffer[SizeOfVector256 / 2]. In
 1871            // that case we can immediately back up to the previous aligned boundary and start the main loop.
 1872            // If the 0x10 bit is *not* set at the pinned address, then it means the 0x10 bit *is* set at
 1873            // address &pAsciiBuffer[SizeOfVector256 / 2], and we should perform one more 16-byte write to bump
 1874            // just past the next aligned boundary address.
 5411875            if (((uint)pAsciiBuffer & (Vector256.Size / 2)) == 0)
 1876            {
 1877                // We need to perform one more partial vector write before we can get the alignment we want.
 1878
 2481879                utf16VectorFirst = Vector256.LoadUnsafe(ref utf16Buffer, currentOffsetInElements);
 1880
 2481881                if (VectorContainsNonAsciiChar(utf16VectorFirst))
 1882                {
 1883                    goto Finish;
 1884                }
 1885
 1886                // Turn the 16 ASCII chars we just read into 16 ASCII bytes, then copy it to the destination.
 2311887                asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 2311888                asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 1889            }
 1890
 1891            // Calculate how many elements we wrote in order to get pAsciiBuffer to its next alignment
 1892            // point, then use that as the base offset going forward.
 1893
 5241894            currentOffsetInElements = Vector256.Size - ((nuint)pAsciiBuffer & MaskOfAllBitsInVector256);
 1895
 5241896            Debug.Assert(0 < currentOffsetInElements && currentOffsetInElements <= Vector256.Size, "We wrote at least 1 
 5241897            Debug.Assert(currentOffsetInElements <= elementCount, "Shouldn't have overrun the destination buffer.");
 5241898            Debug.Assert(elementCount - currentOffsetInElements >= Vector256.Size, "We should be able to run at least on
 1899
 5241900            nuint finalOffsetWhereCanRunLoop = elementCount - Vector256.Size;
 1901            do
 1902            {
 1903                // In a loop, perform two unaligned reads, narrow to a single vector, then aligned write one vector.
 1904
 9551905                utf16VectorFirst = Vector256.LoadUnsafe(ref utf16Buffer, currentOffsetInElements);
 9551906                Vector256<ushort> utf16VectorSecond = Vector256.LoadUnsafe(ref utf16Buffer, currentOffsetInElements + Ve
 9551907                Vector256<ushort> combinedVector = utf16VectorFirst | utf16VectorSecond;
 1908
 9551909                if (VectorContainsNonAsciiChar(combinedVector))
 1910                {
 1911                    goto FoundNonAsciiDataInLoop;
 1912                }
 1913
 1914                // Build up the ASCII vector and perform the store.
 1915
 9391916                Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % Vector256.Size == 0, "Write should be ali
 9391917                asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorSecond);
 9391918                asciiVector.StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 1919
 9391920                currentOffsetInElements += Vector256.Size;
 9391921            } while (currentOffsetInElements <= finalOffsetWhereCanRunLoop);
 1922
 1923        Finish:
 1924
 1925            // There might be some ASCII data left over. That's fine - we'll let our caller handle the final drain.
 5411926            return currentOffsetInElements;
 1927
 1928        FoundNonAsciiDataInLoop:
 1929
 1930            // Can we at least narrow the high vector?
 1931            // See comments in GetIndexOfFirstNonAsciiChar_Intrinsified for information about how this works.
 161932            if (VectorContainsNonAsciiChar(utf16VectorFirst))
 1933            {
 1934                goto Finish;
 1935            }
 1936
 1937            // First part was all ASCII, narrow and aligned write. Note we're only filling in the low half of the vector
 1938
 71939            Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % Vector128.Size == 0, "Destination should be 1
 71940            asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 71941            asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 71942            currentOffsetInElements += Vector256.Size / 2;
 1943
 71944            goto Finish;
 1945        }
 1946
 1947        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 1948        private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_512(char* pUtf16Buffer, byte* pAsciiBuffer, nuint el
 1949        {
 1950            // This method contains logic optimized using vector instructions for x64 only.
 1951            // Much of the logic in this method will be elided by JIT once we determine which specific ISAs we support.
 1952
 1953            // JIT turns the below into constants
 1954
 1955            const nuint MaskOfAllBitsInVector512 = (nuint)(Vector512.Size - 1);
 1956
 1957            // This method is written such that control generally flows top-to-bottom, avoiding
 1958            // jumps as much as possible in the optimistic case of "all ASCII". If we see non-ASCII
 1959            // data, we jump out of the hot paths to targets at the end of the method.
 1960
 8361961            Debug.Assert(Vector512.IsHardwareAccelerated, "Vector512 is required.");
 8361962            Debug.Assert(BitConverter.IsLittleEndian, "This implementation assumes little-endian.");
 8361963            Debug.Assert(elementCount >= 2 * Vector512.Size);
 1964
 1965            // First, perform an unaligned read of the first part of the input buffer.
 8361966            ref ushort utf16Buffer = ref *(ushort*)pUtf16Buffer;
 8361967            Vector512<ushort> utf16VectorFirst = Vector512.LoadUnsafe(ref utf16Buffer);
 1968
 1969            // If there's non-ASCII data in the first 32 elements of the vector, there's nothing we can do.
 8361970            if (VectorContainsNonAsciiChar(utf16VectorFirst))
 1971            {
 391972                return 0;
 1973            }
 1974
 1975            // Turn the 32 ASCII chars we just read into 32 ASCII bytes, then copy it to the destination.
 1976
 7971977            ref byte asciiBuffer = ref *pAsciiBuffer;
 7971978            Vector512<byte> asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 7971979            asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, 0); // how to store the lower part of a avx512
 7971980            nuint currentOffsetInElements = Vector512.Size / 2; // we processed 32 elements so far
 1981
 1982            // We're going to get the best performance when we have aligned writes, so we'll take the
 1983            // hit of potentially unaligned reads in order to hit this sweet spot.
 1984
 1985            // pAsciiBuffer points to the start of the destination buffer, immediately before where we wrote
 1986            // the 32 bytes previously. If the 0x20 bit is set at the pinned address, then the 32 bytes we wrote
 1987            // previously mean that the 0x20 bit is *not* set at address &pAsciiBuffer[SizeOfVector512 / 2]. In
 1988            // that case we can immediately back up to the previous aligned boundary and start the main loop.
 1989            // If the 0x20 bit is *not* set at the pinned address, then it means the 0x20 bit *is* set at
 1990            // address &pAsciiBuffer[SizeOfVector512 / 2], and we should perform one more 32-byte write to bump
 1991            // just past the next aligned boundary address.
 1992
 7971993            if (((uint)pAsciiBuffer & (Vector512.Size / 2)) == 0)
 1994            {
 1995                // We need to perform one more partial vector write before we can get the alignment we want.
 1996
 3601997                utf16VectorFirst = Vector512.LoadUnsafe(ref utf16Buffer, currentOffsetInElements);
 1998
 3601999                if (VectorContainsNonAsciiChar(utf16VectorFirst))
 2000                {
 2001                    goto Finish;
 2002                }
 2003
 2004                // Turn the 32 ASCII chars we just read into 32 ASCII bytes, then copy it to the destination.
 3582005                asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 3582006                asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 2007            }
 2008
 2009            // Calculate how many elements we wrote in order to get pAsciiBuffer to its next alignment
 2010            // point, then use that as the base offset going forward.
 2011
 7952012            currentOffsetInElements = Vector512.Size - ((nuint)pAsciiBuffer & MaskOfAllBitsInVector512);
 2013
 7952014            Debug.Assert(0 < currentOffsetInElements && currentOffsetInElements <= Vector512.Size, "We wrote at least 1 
 7952015            Debug.Assert(currentOffsetInElements <= elementCount, "Shouldn't have overrun the destination buffer.");
 7952016            Debug.Assert(elementCount - currentOffsetInElements >= Vector512.Size, "We should be able to run at least on
 2017
 7952018            nuint finalOffsetWhereCanRunLoop = elementCount - Vector512.Size;
 2019            do
 2020            {
 2021                // In a loop, perform two unaligned reads, narrow to a single vector, then aligned write one vector.
 2022
 27992023                utf16VectorFirst = Vector512.LoadUnsafe(ref utf16Buffer, currentOffsetInElements);
 27992024                Vector512<ushort> utf16VectorSecond = Vector512.LoadUnsafe(ref utf16Buffer, currentOffsetInElements + Ve
 27992025                Vector512<ushort> combinedVector = utf16VectorFirst | utf16VectorSecond;
 2026
 27992027                if (VectorContainsNonAsciiChar(combinedVector))
 2028                {
 2029                    goto FoundNonAsciiDataInLoop;
 2030                }
 2031
 2032                // Build up the ASCII vector and perform the store.
 2033
 27602034                Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % Vector512.Size == 0, "Write should be ali
 27602035                asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorSecond);
 27602036                asciiVector.StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 2037
 27602038                currentOffsetInElements += Vector512.Size;
 27602039            } while (currentOffsetInElements <= finalOffsetWhereCanRunLoop);
 2040
 2041        Finish:
 2042
 2043            // There might be some ASCII data left over. That's fine - we'll let our caller handle the final drain.
 7972044            return currentOffsetInElements;
 2045
 2046        FoundNonAsciiDataInLoop:
 2047
 2048            // Can we at least narrow the high vector?
 2049            // See comments in GetIndexOfFirstNonAsciiChar_Intrinsified for information about how this works.
 392050            if (VectorContainsNonAsciiChar(utf16VectorFirst))
 2051            {
 2052                goto Finish;
 2053            }
 2054
 2055            // First part was all ASCII, narrow and aligned write. Note we're only filling in the low half of the vector
 2056
 192057            Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % Vector256.Size == 0, "Destination should be 2
 192058            asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
 192059            asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 192060            currentOffsetInElements += Vector512.Size / 2;
 2061
 192062            goto Finish;
 2063        }
 2064#endif
 2065
 2066        /// <summary>
 2067        /// Copies as many ASCII bytes (00..7F) as possible from <paramref name="pAsciiBuffer"/>
 2068        /// to <paramref name="pUtf16Buffer"/>, stopping when the first non-ASCII byte is encountered
 2069        /// or once <paramref name="elementCount"/> elements have been converted. Returns the total number
 2070        /// of elements that were able to be converted.
 2071        /// </summary>
 2072        internal static unsafe nuint WidenAsciiToUtf16(byte* pAsciiBuffer, char* pUtf16Buffer, nuint elementCount)
 2073        {
 2074            // Intrinsified in mono interpreter
 2075            nuint currentOffset = 0;
 2076
 2077#if NET
 55719202078            if (BitConverter.IsLittleEndian && Vector128.IsHardwareAccelerated && elementCount >= (uint)Vector128<byte>.
 2079            {
 24533992080                if (Vector512.IsHardwareAccelerated && (elementCount - currentOffset) >= (uint)Vector512<byte>.Count)
 2081                {
 15779772082                    WidenAsciiToUtf1_Vector<Vector512<byte>, Vector512<ushort>>(pAsciiBuffer, pUtf16Buffer, ref currentO
 2083                }
 8754222084                else if (Vector256.IsHardwareAccelerated && (elementCount - currentOffset) >= (uint)Vector256<byte>.Coun
 2085                {
 5034692086                    WidenAsciiToUtf1_Vector<Vector256<byte>, Vector256<ushort>>(pAsciiBuffer, pUtf16Buffer, ref currentO
 2087                }
 3719532088                else if (Vector128.IsHardwareAccelerated && (elementCount - currentOffset) >= (uint)Vector128<byte>.Coun
 2089                {
 3719532090                    WidenAsciiToUtf1_Vector<Vector128<byte>, Vector128<ushort>>(pAsciiBuffer, pUtf16Buffer, ref currentO
 2091                }
 2092            }
 2093#endif
 2094
 55719202095            Debug.Assert(currentOffset <= elementCount);
 55719202096            nuint remainingElementCount = elementCount - currentOffset;
 2097
 2098            // Try to widen 32 bits -> 64 bits at a time.
 2099            // We needn't update remainingElementCount after this point.
 2100
 2101            uint asciiData;
 2102
 55719202103            if (remainingElementCount >= 4)
 2104            {
 30731222105                nuint finalOffsetWhereCanLoop = currentOffset + remainingElementCount - 4;
 2106                do
 2107                {
 33328712108                    asciiData = Unsafe.ReadUnaligned<uint>(pAsciiBuffer + currentOffset);
 33328712109                    if (!AllBytesInUInt32AreAscii(asciiData))
 2110                    {
 2111                        goto FoundNonAsciiData;
 2112                    }
 2113
 3190162114                    WidenFourAsciiBytesToUtf16AndWriteToBuffer(ref pUtf16Buffer[currentOffset], asciiData);
 3190162115                    currentOffset += 4;
 3190162116                } while (currentOffset <= finalOffsetWhereCanLoop);
 2117            }
 2118
 2119            // Try to widen 16 bits -> 32 bits.
 2120
 25580652121            if (((uint)remainingElementCount & 2) != 0)
 2122            {
 11073832123                asciiData = Unsafe.ReadUnaligned<ushort>(pAsciiBuffer + currentOffset);
 11073832124                if (!AllBytesInUInt32AreAscii(asciiData))
 2125                {
 9052622126                    if (!BitConverter.IsLittleEndian)
 2127                    {
 2128                        asciiData <<= 16;
 2129                    }
 2130                    goto FoundNonAsciiData;
 2131                }
 2132
 2021212133                if (BitConverter.IsLittleEndian)
 2134                {
 2021212135                    pUtf16Buffer[currentOffset] = (char)(byte)asciiData;
 2021212136                    pUtf16Buffer[currentOffset + 1] = (char)(asciiData >> 8);
 2137                }
 2138                else
 2139                {
 2140                    pUtf16Buffer[currentOffset + 1] = (char)(byte)asciiData;
 2141                    pUtf16Buffer[currentOffset] = (char)(asciiData >> 8);
 2142                }
 2143
 2021212144                currentOffset += 2;
 2145            }
 2146
 2147            // Try to widen 8 bits -> 16 bits.
 2148
 16528032149            if (((uint)remainingElementCount & 1) != 0)
 2150            {
 14436102151                asciiData = pAsciiBuffer[currentOffset];
 14436102152                if (((byte)asciiData & 0x80) != 0)
 2153                {
 2154                    goto Finish;
 2155                }
 2156
 4022962157                pUtf16Buffer[currentOffset] = (char)asciiData;
 4022962158                currentOffset++;
 2159            }
 2160
 2161        Finish:
 2162
 55719202163            return currentOffset;
 2164
 2165        FoundNonAsciiData:
 2166
 39191172167            Debug.Assert(!AllBytesInUInt32AreAscii(asciiData), "Shouldn't have reached this point if we have an all-ASCI
 2168
 2169            // Drain ASCII bytes one at a time.
 2170
 39191172171            if (BitConverter.IsLittleEndian)
 2172            {
 47021722173                while (((byte)asciiData & 0x80) == 0)
 2174                {
 7830552175                    pUtf16Buffer[currentOffset] = (char)(byte)asciiData;
 7830552176                    currentOffset++;
 7830552177                    asciiData >>= 8;
 2178                }
 2179            }
 2180            else
 2181            {
 2182                while ((asciiData & 0x80000000) == 0)
 2183                {
 2184                    asciiData = BitOperations.RotateLeft(asciiData, 8);
 2185                    pUtf16Buffer[currentOffset] = (char)(byte)asciiData;
 2186                    currentOffset++;
 2187                }
 2188            }
 2189
 2190            goto Finish;
 2191        }
 2192
 2193#if NET
 2194        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 2195        private static unsafe void WidenAsciiToUtf1_Vector<TVectorByte, TVectorUInt16>(byte* pAsciiBuffer, char* pUtf16B
 2196            where TVectorByte : unmanaged, ISimdVector<TVectorByte, byte>
 2197            where TVectorUInt16 : unmanaged, ISimdVector<TVectorUInt16, ushort>
 2198        {
 24533992199            ushort* pCurrentWriteAddress = (ushort*)pUtf16Buffer;
 2200            // Calculating the destination address outside the loop results in significant
 2201            // perf wins vs. relying on the JIT to fold memory addressing logic into the
 2202            // write instructions. See: https://github.com/dotnet/runtime/issues/33002
 24533992203            nuint finalOffsetWhereCanRunLoop = elementCount - (nuint)TVectorByte.ElementCount;
 24533992204            TVectorByte asciiVector = TVectorByte.Load(pAsciiBuffer + currentOffset);
 24533992205            if (!HasMatch<TVectorByte>(asciiVector))
 2206            {
 145842207                (TVectorUInt16 utf16LowVector, TVectorUInt16 utf16HighVector) = Widen<TVectorByte, TVectorUInt16>(asciiV
 145842208                utf16LowVector.Store(pCurrentWriteAddress);
 145842209                utf16HighVector.Store(pCurrentWriteAddress + TVectorUInt16.ElementCount);
 145842210                pCurrentWriteAddress += (nuint)(TVectorUInt16.ElementCount * 2);
 145842211                if (((nuint)pCurrentWriteAddress % sizeof(char)) == 0)
 2212                {
 2213                    // Bump write buffer up to the next aligned boundary
 145842214                    pCurrentWriteAddress = (ushort*)((nuint)pCurrentWriteAddress & ~(nuint)(TVectorUInt16.Alignment - 1)
 145842215                    nuint numBytesWritten = (nuint)pCurrentWriteAddress - (nuint)pUtf16Buffer;
 145842216                    currentOffset += (nuint)numBytesWritten / 2;
 2217                }
 2218                else
 2219                {
 2220                    // If input isn't char aligned, we won't be able to align it to a Vector
 02221                    currentOffset += (nuint)TVectorByte.ElementCount;
 2222                }
 241132223                while (currentOffset <= finalOffsetWhereCanRunLoop)
 2224                {
 133232225                    asciiVector = TVectorByte.Load(pAsciiBuffer + currentOffset);
 133232226                    if (HasMatch<TVectorByte>(asciiVector))
 2227                    {
 2228                        break;
 2229                    }
 95292230                    (utf16LowVector, utf16HighVector) = Widen<TVectorByte, TVectorUInt16>(asciiVector);
 95292231                    utf16LowVector.Store(pCurrentWriteAddress);
 95292232                    utf16HighVector.Store(pCurrentWriteAddress + TVectorUInt16.ElementCount);
 2233
 95292234                    currentOffset += (nuint)TVectorByte.ElementCount;
 95292235                    pCurrentWriteAddress += (nuint)(TVectorUInt16.ElementCount * 2);
 2236                }
 2237            }
 24533992238            return;
 2239        }
 2240
 2241        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 2242        private static bool HasMatch<TVectorByte>(TVectorByte vector)
 2243            where TVectorByte : unmanaged, ISimdVector<TVectorByte, byte>
 2244        {
 24667222245            return !(vector & TVectorByte.Create((byte)0x80)).Equals(TVectorByte.Zero);
 2246        }
 2247
 2248
 2249        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 2250        private static (TVectorUInt16 Lower, TVectorUInt16 Upper) Widen<TVectorByte, TVectorUInt16>(TVectorByte vector)
 2251            where TVectorByte : unmanaged, ISimdVector<TVectorByte, byte>
 2252            where TVectorUInt16 : unmanaged, ISimdVector<TVectorUInt16, ushort>
 2253        {
 241132254            if (typeof(TVectorByte) == typeof(Vector256<byte>))
 2255            {
 44752256                (Vector256<ushort> Lower256, Vector256<ushort> Upper256) = Vector256.Widen((Vector256<byte>)(object)vect
 44752257                return ((TVectorUInt16)(object)Lower256, (TVectorUInt16)(object)Upper256);
 2258            }
 196382259            else if (typeof(TVectorByte) == typeof(Vector512<byte>))
 2260            {
 154002261                (Vector512<ushort> Lower512, Vector512<ushort> Upper512) = Vector512.Widen((Vector512<byte>)(object)vect
 154002262                return ((TVectorUInt16)(object)Lower512, (TVectorUInt16)(object)Upper512);
 2263            }
 2264            else
 2265            {
 42382266                Debug.Assert(typeof(TVectorByte) == typeof(Vector128<byte>));
 42382267                (Vector128<ushort> Lower128, Vector128<ushort> Upper128) = Vector128.Widen((Vector128<byte>)(object)vect
 42382268                return ((TVectorUInt16)(object)Lower128, (TVectorUInt16)(object)Upper128);
 2269            }
 2270        }
 2271#endif
 2272
 2273        /// <summary>
 2274        /// Given a DWORD which represents a buffer of 4 bytes, widens the buffer into 4 WORDs and
 2275        /// writes them to the output buffer with machine endianness.
 2276        /// </summary>
 2277        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 2278        internal static void WidenFourAsciiBytesToUtf16AndWriteToBuffer(ref char outputBuffer, uint value)
 2279        {
 2280            Debug.Assert(AllBytesInUInt32AreAscii(value));
 2281
 2282#if NET
 2283            if (AdvSimd.Arm64.IsSupported)
 2284            {
 2285                Vector128<byte> vecNarrow = AdvSimd.DuplicateToVector128(value).AsByte();
 2286                Vector128<ulong> vecWide = AdvSimd.Arm64.ZipLow(vecNarrow, Vector128<byte>.Zero).AsUInt64();
 2287                Unsafe.WriteUnaligned(ref Unsafe.As<char, byte>(ref outputBuffer), vecWide.ToScalar());
 2288            }
 3807302289            else if (Vector128.IsHardwareAccelerated)
 2290            {
 3807302291                Vector128<byte> vecNarrow = Vector128.CreateScalar(value).AsByte();
 3807302292                Vector128<ulong> vecWide = Vector128.WidenLower(vecNarrow).AsUInt64();
 3807302293                Unsafe.WriteUnaligned(ref Unsafe.As<char, byte>(ref outputBuffer), vecWide.ToScalar());
 2294            }
 2295            else
 2296#endif
 2297            {
 02298                if (BitConverter.IsLittleEndian)
 2299                {
 02300                    outputBuffer = (char)(byte)value;
 02301                    value >>= 8;
 02302                    Unsafe.Add(ref outputBuffer, 1) = (char)(byte)value;
 02303                    value >>= 8;
 02304                    Unsafe.Add(ref outputBuffer, 2) = (char)(byte)value;
 02305                    value >>= 8;
 02306                    Unsafe.Add(ref outputBuffer, 3) = (char)value;
 2307                }
 2308                else
 2309                {
 2310                    Unsafe.Add(ref outputBuffer, 3) = (char)(byte)value;
 2311                    value >>= 8;
 2312                    Unsafe.Add(ref outputBuffer, 2) = (char)(byte)value;
 2313                    value >>= 8;
 2314                    Unsafe.Add(ref outputBuffer, 1) = (char)(byte)value;
 2315                    value >>= 8;
 2316                    outputBuffer = (char)value;
 2317                }
 2318            }
 2319        }
 2320    }
 2321}
 2322

https://raw.githubusercontent.com/dotnet/runtime/811a7eabb75c42db53440e8ba3f60c07511cfd1f/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.Helpers.cs

#LineLine coverage
 1// Licensed to the .NET Foundation under one or more agreements.
 2// The .NET Foundation licenses this file to you under the MIT license.
 3
 4using System.Diagnostics;
 5using System.Numerics;
 6using System.Runtime.CompilerServices;
 7using System.Runtime.InteropServices;
 8
 9namespace System.Text
 10{
 11#if SYSTEM_PRIVATE_CORELIB
 12    public
 13#else
 14    internal
 15#endif
 16        static partial class Ascii
 17    {
 18        /// <summary>
 19        /// A mask which selects only the high bit of each byte of the given <see cref="uint"/>.
 20        /// </summary>
 21        private const uint UInt32HighBitsOnlyMask = 0x80808080u;
 22
 23        /// <summary>
 24        /// A mask which selects only the high bit of each byte of the given <see cref="ulong"/>.
 25        /// </summary>
 26        private const ulong UInt64HighBitsOnlyMask = 0x80808080_80808080ul;
 27
 28        /// <summary>
 29        /// Returns <see langword="true"/> iff all bytes in <paramref name="value"/> are ASCII.
 30        /// </summary>
 31        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 32        internal static bool AllBytesInUInt32AreAscii(uint value)
 33        {
 34            // If the high bit of any byte is set, that byte is non-ASCII.
 35
 1740219436            return (value & UInt32HighBitsOnlyMask) == 0;
 37        }
 38
 39        /// <summary>
 40        /// Given a DWORD which represents a four-byte buffer read in machine endianness, and which
 41        /// the caller has asserted contains a non-ASCII byte *somewhere* in the data, counts the
 42        /// number of consecutive ASCII bytes starting from the beginning of the buffer. Returns
 43        /// a value 0 - 3, inclusive. (The caller is responsible for ensuring that the buffer doesn't
 44        /// contain all-ASCII data.)
 45        /// </summary>
 46        [MethodImpl(MethodImplOptions.AggressiveInlining)]
 47        internal static uint CountNumberOfLeadingAsciiBytesFromUInt32WithSomeNonAsciiData(uint value)
 48        {
 49            Debug.Assert(!AllBytesInUInt32AreAscii(value), "Caller shouldn't provide an all-ASCII value.");
 50
 172025851            if (BitConverter.IsLittleEndian)
 52            {
 172025853                return (uint)BitOperations.TrailingZeroCount(value & UInt32HighBitsOnlyMask) >> 3;
 54            }
 55            else
 56            {
 57                // Couldn't use tzcnt, use specialized software fallback.
 58                // The 'allBytesUpToNowAreAscii' DWORD uses bit twiddling to hold a 1 or a 0 depending
 59                // on whether all processed bytes were ASCII. Then we accumulate all of the
 60                // results to calculate how many consecutive ASCII bytes are present.
 61
 62                value = ~value;
 63
 64                // BinaryPrimitives.ReverseEndianness is only implemented as an intrinsic on
 65                // little-endian platforms, so using it in this big-endian path would be too
 66                // expensive. Instead we'll just change how we perform the shifts.
 67
 68                // Read first byte
 69                value = BitOperations.RotateLeft(value, 1);
 70                uint allBytesUpToNowAreAscii = value & 1;
 71                uint numAsciiBytes = allBytesUpToNowAreAscii;
 72
 73                // Read second byte
 74                value = BitOperations.RotateLeft(value, 8);
 75                allBytesUpToNowAreAscii &= value;
 76                numAsciiBytes += allBytesUpToNowAreAscii;
 77
 78                // Read third byte
 79                value = BitOperations.RotateLeft(value, 8);
 80                allBytesUpToNowAreAscii &= value;
 81                numAsciiBytes += allBytesUpToNowAreAscii;
 82
 83                return numAsciiBytes;
 84            }
 85        }
 86    }
 87}
 88
Generated by: ReportGenerator 5.5.11.0
10/9/2026 - 5:58:09 PM
GitHub | reportgenerator.io

Methods/Properties

ToUpper(System.ReadOnlySpan`1<System.Byte>,System.Span`1<System.Byte>,System.Int32&)
ToUpper(System.ReadOnlySpan`1<System.Char>,System.Span`1<System.Char>,System.Int32&)
ToUpper(System.ReadOnlySpan`1<System.Byte>,System.Span`1<System.Char>,System.Int32&)
ToUpper(System.ReadOnlySpan`1<System.Char>,System.Span`1<System.Byte>,System.Int32&)
ToLower(System.ReadOnlySpan`1<System.Byte>,System.Span`1<System.Byte>,System.Int32&)
ToLower(System.ReadOnlySpan`1<System.Char>,System.Span`1<System.Char>,System.Int32&)
ToLower(System.ReadOnlySpan`1<System.Byte>,System.Span`1<System.Char>,System.Int32&)
ToLower(System.ReadOnlySpan`1<System.Char>,System.Span`1<System.Byte>,System.Int32&)
ToLowerInPlace(System.Span`1<System.Byte>,System.Int32&)
ToLowerInPlace(System.Span`1<System.Char>,System.Int32&)
ToUpperInPlace(System.Span`1<System.Byte>,System.Int32&)
ToUpperInPlace(System.Span`1<System.Char>,System.Int32&)
ChangeCase(System.ReadOnlySpan`1<TFrom>,System.Span`1<TTo>,System.Int32&)
ChangeCase(System.Span`1<T>,System.Int32&)
ChangeCase(TFrom*,TTo*,System.UIntPtr)
ChangeWidthAndWriteTo(System.Runtime.Intrinsics.Vector128`1<TFrom>,TTo*,System.UIntPtr)
SignedLessThan(System.Runtime.Intrinsics.Vector128`1<T>,System.Runtime.Intrinsics.Vector128`1<T>)
IsValid(System.ReadOnlySpan`1<System.Byte>)
IsValid(System.ReadOnlySpan`1<System.Char>)
IsValid(System.Byte)
IsValid(System.Char)
IsValidCore(T&,System.Int32)
Equals(System.ReadOnlySpan`1<System.Byte>,System.ReadOnlySpan`1<System.Byte>)
Equals(System.ReadOnlySpan`1<System.Byte>,System.ReadOnlySpan`1<System.Char>)
Equals(System.ReadOnlySpan`1<System.Char>,System.ReadOnlySpan`1<System.Byte>)
Equals(System.ReadOnlySpan`1<System.Char>,System.ReadOnlySpan`1<System.Char>)
Equals(TLeft&,TRight&,System.UIntPtr)
EqualsIgnoreCase(System.ReadOnlySpan`1<System.Byte>,System.ReadOnlySpan`1<System.Byte>)
EqualsIgnoreCase(System.ReadOnlySpan`1<System.Byte>,System.ReadOnlySpan`1<System.Char>)
EqualsIgnoreCase(System.ReadOnlySpan`1<System.Char>,System.ReadOnlySpan`1<System.Byte>)
EqualsIgnoreCase(System.ReadOnlySpan`1<System.Char>,System.ReadOnlySpan`1<System.Char>)
EqualsIgnoreCase(System.Char&,System.Char&,System.UIntPtr)
EqualsIgnoreCase(TLeft&,TRight&,System.UIntPtr)
Load128(T&)
Load256(T&)
Load512(T&)
EqualAndAscii256(T&,T&)
EqualAndAscii512(T&,T&)
Load128(System.Byte&)
Load256(System.Byte&)
Load512(System.Byte&)
EqualAndAscii256(System.Byte&,System.UInt16&)
EqualAndAscii512(System.Byte&,System.UInt16&)
ToUtf16(System.ReadOnlySpan`1<System.Byte>,System.Span`1<System.Char>,System.Int32&)
FromUtf16(System.ReadOnlySpan`1<System.Char>,System.Span`1<System.Byte>,System.Int32&)
Trim(System.ReadOnlySpan`1<System.Byte>)
Trim(System.ReadOnlySpan`1<System.Char>)
TrimStart(System.ReadOnlySpan`1<System.Byte>)
TrimStart(System.ReadOnlySpan`1<System.Char>)
TrimEnd(System.ReadOnlySpan`1<System.Byte>)
TrimEnd(System.ReadOnlySpan`1<System.Char>)
TrimHelper(System.ReadOnlySpan`1<T>,System.Text.TrimType)
AllBytesInUInt64AreAscii(System.UInt64)
AllCharsInUInt32AreAscii(System.UInt32)
AllCharsInUInt64AreAscii(System.UInt64)
GetIndexOfFirstNonAsciiByte(System.Byte*,System.UIntPtr)
GetIndexOfFirstNonAsciiByte_Vector(System.Byte*,System.UIntPtr)
ContainsNonAsciiByte_Sse2(System.UInt32)
GetIndexOfFirstNonAsciiByte_Intrinsified(System.Byte*,System.UIntPtr)
GetIndexOfFirstNonAsciiChar(System.Char*,System.UIntPtr)
GetIndexOfFirstNonAsciiChar_Vector(System.Char*,System.UIntPtr)
GetIndexOfFirstNonAsciiChar_Intrinsified(System.Char*,System.UIntPtr)
NarrowFourUtf16CharsToAsciiAndWriteToBuffer(System.Byte&,System.UInt64)
NarrowTwoUtf16CharsToAsciiAndWriteToBuffer(System.Byte&,System.UInt32)
NarrowUtf16ToAscii(System.Char*,System.Byte*,System.UIntPtr)
VectorContainsNonAsciiChar(System.Runtime.Intrinsics.Vector128`1<System.Byte>)
VectorContainsNonAsciiChar(System.Runtime.Intrinsics.Vector128`1<System.UInt16>)
VectorContainsNonAsciiChar(System.Runtime.Intrinsics.Vector256`1<System.UInt16>)
VectorContainsNonAsciiChar(System.Runtime.Intrinsics.Vector512`1<System.UInt16>)
VectorContainsNonAsciiChar(System.Runtime.Intrinsics.Vector128`1<T>)
AllCharsInVectorAreAscii(System.Runtime.Intrinsics.Vector128`1<T>)
AllCharsInVectorAreAscii(System.Runtime.Intrinsics.Vector256`1<T>)
AllCharsInVectorAreAscii(System.Runtime.Intrinsics.Vector512`1<T>)
ExtractAsciiVector(System.Runtime.Intrinsics.Vector128`1<System.UInt16>,System.Runtime.Intrinsics.Vector128`1<System.UInt16>)
ExtractAsciiVector(System.Runtime.Intrinsics.Vector256`1<System.UInt16>,System.Runtime.Intrinsics.Vector256`1<System.UInt16>)
ExtractAsciiVector(System.Runtime.Intrinsics.Vector512`1<System.UInt16>,System.Runtime.Intrinsics.Vector512`1<System.UInt16>)
NarrowUtf16ToAscii_Intrinsified(System.Char*,System.Byte*,System.UIntPtr)
NarrowUtf16ToAscii_Intrinsified_256(System.Char*,System.Byte*,System.UIntPtr)
NarrowUtf16ToAscii_Intrinsified_512(System.Char*,System.Byte*,System.UIntPtr)
WidenAsciiToUtf16(System.Byte*,System.Char*,System.UIntPtr)
WidenAsciiToUtf1_Vector(System.Byte*,System.Char*,System.UIntPtr&,System.UIntPtr)
HasMatch(TVectorByte)
Widen(TVectorByte)
WidenFourAsciiBytesToUtf16AndWriteToBuffer(System.Char&,System.UInt32)
AllBytesInUInt32AreAscii(System.UInt32)
CountNumberOfLeadingAsciiBytesFromUInt32WithSomeNonAsciiData(System.UInt32)