| | | 1 | | // Licensed to the .NET Foundation under one or more agreements. |
| | | 2 | | // The .NET Foundation licenses this file to you under the MIT license. |
| | | 3 | | |
| | | 4 | | using System.ComponentModel; |
| | | 5 | | using System.Buffers; |
| | | 6 | | using System.Collections.Generic; |
| | | 7 | | using System.Collections; |
| | | 8 | | |
| | | 9 | | namespace System.Text; |
| | | 10 | | |
| | | 11 | | /// <summary> |
| | | 12 | | /// Represents a position in Unicode data, allowing for deeper data inspection. |
| | | 13 | | /// </summary> |
| | | 14 | | /// <remarks> |
| | | 15 | | /// Invalid Unicode symbols will be represented by the <see cref="System.Text.Rune.ReplacementChar"/> value. |
| | | 16 | | /// </remarks> |
| | | 17 | | public readonly struct RunePosition : IEquatable<RunePosition> |
| | | 18 | | { |
| | | 19 | | /// <summary> |
| | | 20 | | /// Returns an enumeration of <see cref="RunePosition"/> from the provided span that allows deeper data inspection. |
| | | 21 | | /// </summary> |
| | | 22 | | /// <param name="span">The <see cref="ReadOnlySpan{T}"/> with Unicode data.</param> |
| | | 23 | | /// <returns> |
| | | 24 | | /// <see cref="Utf16Enumerator"/> to enumerate <see cref="RunePosition"/> from the provided span with UTF-16 |
| | | 25 | | /// Unicode data. |
| | | 26 | | /// </returns> |
| | | 27 | | /// <remarks> |
| | | 28 | | /// Invalid Unicode symbols will be represented by <see cref="System.Text.Rune.ReplacementChar"/> |
| | | 29 | | /// value. |
| | | 30 | | /// </remarks> |
| | 0 | 31 | | public static Utf16Enumerator EnumerateUtf16(ReadOnlySpan<char> span) => new(span); |
| | | 32 | | |
| | | 33 | | /// <summary> |
| | | 34 | | /// Returns an enumeration of <see cref="RunePosition"/> from the provided span that allows deeper data inspection. |
| | | 35 | | /// </summary> |
| | | 36 | | /// <param name="span">The <see cref="ReadOnlySpan{T}"/> with Unicode data.</param> |
| | | 37 | | /// <returns> |
| | | 38 | | /// <see cref="Utf8Enumerator"/> to enumerate <see cref="RunePosition"/> from the provided span with UTF-8 Unicode |
| | | 39 | | /// data. |
| | | 40 | | /// </returns> |
| | | 41 | | /// <remarks> |
| | | 42 | | /// Invalid Unicode symbols will be represented by <see cref="Rune.ReplacementChar"/> value. |
| | | 43 | | /// </remarks> |
| | 0 | 44 | | public static Utf8Enumerator EnumerateUtf8(ReadOnlySpan<byte> span) => new(span); |
| | | 45 | | |
| | | 46 | | /// <summary> |
| | | 47 | | /// Unicode scalar value <see cref="System.Text.Rune"/> of the current symbol in Unicode data. |
| | | 48 | | /// Invalid Unicode symbols will be represented by <see cref="System.Text.Rune.ReplacementChar"/> value. |
| | | 49 | | /// </summary> |
| | 0 | 50 | | public Rune Rune { get; } |
| | | 51 | | |
| | | 52 | | /// <summary> |
| | | 53 | | /// The index of current symbol in Unicode data. |
| | | 54 | | /// </summary> |
| | 0 | 55 | | public int StartIndex { get; } |
| | | 56 | | |
| | | 57 | | /// <summary> |
| | | 58 | | /// The length of current symbol in Unicode data. |
| | | 59 | | /// </summary> |
| | 0 | 60 | | public int Length { get; } |
| | | 61 | | |
| | | 62 | | /// <summary> |
| | | 63 | | /// <see langword="false"/> it current Unicode symbol is correct encoded and <see cref="RunePosition.Rune"/> |
| | | 64 | | /// contain its scalar value. |
| | | 65 | | /// <br /> |
| | | 66 | | /// <see langword="true"/> if current Unicode symbol is invalid encoded and <see cref="RunePosition.Rune"/> was |
| | | 67 | | /// replaced by <see cref="System.Text.Rune.ReplacementChar"/> value. |
| | | 68 | | /// </summary> |
| | 0 | 69 | | public bool WasReplaced { get; } |
| | | 70 | | |
| | | 71 | | /// <summary> |
| | | 72 | | /// Initializes a new instance of the <see cref="RunePosition"/> struct. |
| | | 73 | | /// </summary> |
| | | 74 | | /// <param name="rune">The Unicode scalar value.</param> |
| | | 75 | | /// <param name="startIndex">The index of the current symbol in Unicode data.</param> |
| | | 76 | | /// <param name="length">The length of the current symbol in Unicode data.</param> |
| | | 77 | | /// <param name="wasReplaced">Indicates if the current Unicode symbol was replaced.</param> |
| | | 78 | | public RunePosition(Rune rune, int startIndex, int length, bool wasReplaced) |
| | | 79 | | { |
| | 0 | 80 | | if (startIndex < 0) |
| | | 81 | | { |
| | 0 | 82 | | throw new ArgumentOutOfRangeException(nameof(startIndex), SR.ArgumentOutOfRange_NeedNonNegNum); |
| | | 83 | | } |
| | | 84 | | |
| | 0 | 85 | | if ((uint)length > Rune.MaxUtf8BytesPerRune) |
| | | 86 | | { |
| | 0 | 87 | | throw new ArgumentOutOfRangeException(nameof(length), SR.ArgumentOutOfRange_NeedNonNegNum); |
| | | 88 | | } |
| | | 89 | | |
| | 0 | 90 | | StartIndex = startIndex; |
| | 0 | 91 | | Length = length; |
| | 0 | 92 | | Rune = rune; |
| | 0 | 93 | | WasReplaced = wasReplaced; |
| | 0 | 94 | | } |
| | | 95 | | |
| | | 96 | | /// <summary> |
| | | 97 | | /// Determines whether the specified <see cref="RunePosition"/> is equal to the current <see cref="RunePosition"/>. |
| | | 98 | | /// </summary> |
| | | 99 | | /// <param name="other">The other <see cref="RunePosition"/> to compare with.</param> |
| | | 100 | | /// <returns> |
| | | 101 | | /// <see langword="true"/> if the specified <see cref="RunePosition"/> is equal to the current |
| | | 102 | | /// <see cref="RunePosition"/>; otherwise, <see langword="false"/>. |
| | | 103 | | /// </returns> |
| | | 104 | | public bool Equals(RunePosition other) => |
| | 0 | 105 | | Rune == other.Rune && StartIndex == other.StartIndex && Length == other.Length && WasReplaced == other.WasReplac |
| | | 106 | | |
| | | 107 | | /// <summary> |
| | | 108 | | /// Determines whether the specified <see cref="object"/> is equal to the current <see cref="RunePosition"/>. |
| | | 109 | | /// </summary> |
| | | 110 | | /// <param name="obj">The object to compare with the current <see cref="RunePosition"/>.</param> |
| | | 111 | | /// <returns> |
| | | 112 | | /// <see langword="true"/> if the specified <see cref="object"/> is equal to the current |
| | | 113 | | /// <see cref="RunePosition"/>; otherwise, <see langword="false"/>. |
| | | 114 | | /// </returns> |
| | | 115 | | public override bool Equals(object? obj) => |
| | 0 | 116 | | obj is RunePosition runePosition && Equals(runePosition); |
| | | 117 | | |
| | | 118 | | /// <summary> |
| | | 119 | | /// Returns the hash code for the current <see cref="RunePosition"/>. |
| | | 120 | | /// </summary> |
| | | 121 | | /// <returns>The hash code for the current <see cref="RunePosition"/>.</returns> |
| | | 122 | | public override int GetHashCode() => |
| | 0 | 123 | | HashCode.Combine(Rune, StartIndex, Length, WasReplaced); |
| | | 124 | | |
| | | 125 | | /// <summary> |
| | | 126 | | /// Deconstructs the <see cref="RunePosition"/> into its components. |
| | | 127 | | /// </summary> |
| | | 128 | | /// <param name="rune">The Unicode scalar value.</param> |
| | | 129 | | /// <param name="startIndex">The index of the current symbol in Unicode data.</param> |
| | | 130 | | [EditorBrowsable(EditorBrowsableState.Never)] |
| | | 131 | | public void Deconstruct(out Rune rune, out int startIndex) |
| | | 132 | | { |
| | 0 | 133 | | rune = Rune; |
| | 0 | 134 | | startIndex = StartIndex; |
| | 0 | 135 | | } |
| | | 136 | | |
| | | 137 | | /// <summary> |
| | | 138 | | /// Deconstructs the <see cref="RunePosition"/> into its components. |
| | | 139 | | /// </summary> |
| | | 140 | | /// <param name="rune">The Unicode scalar value.</param> |
| | | 141 | | /// <param name="startIndex">The index of the current symbol in Unicode data.</param> |
| | | 142 | | /// <param name="length">The length of the current symbol in Unicode data.</param> |
| | | 143 | | [EditorBrowsable(EditorBrowsableState.Never)] |
| | | 144 | | public void Deconstruct(out Rune rune, out int startIndex, out int length) |
| | | 145 | | { |
| | 0 | 146 | | rune = Rune; |
| | 0 | 147 | | startIndex = StartIndex; |
| | 0 | 148 | | length = Length; |
| | 0 | 149 | | } |
| | | 150 | | |
| | | 151 | | /// <summary> |
| | | 152 | | /// Determines whether two specified <see cref="RunePosition"/> instances are equal. |
| | | 153 | | /// </summary> |
| | | 154 | | /// <param name="left">The first <see cref="RunePosition"/> to compare.</param> |
| | | 155 | | /// <param name="right">The second <see cref="RunePosition"/> to compare.</param> |
| | | 156 | | /// <returns> |
| | | 157 | | /// <see langword="true"/> if the two <see cref="RunePosition"/> instances are equal; otherwise, |
| | | 158 | | /// <see langword="false"/>. |
| | | 159 | | /// </returns> |
| | 0 | 160 | | public static bool operator ==(RunePosition left, RunePosition right) => left.Equals(right); |
| | | 161 | | |
| | | 162 | | /// <summary> |
| | | 163 | | /// Determines whether two specified <see cref="RunePosition"/> instances are not equal. |
| | | 164 | | /// </summary> |
| | | 165 | | /// <param name="left">The first <see cref="RunePosition"/> to compare.</param> |
| | | 166 | | /// <param name="right">The second <see cref="RunePosition"/> to compare.</param> |
| | | 167 | | /// <returns> |
| | | 168 | | /// <see langword="true"/> if the two <see cref="RunePosition"/> instances are not equal; otherwise, |
| | | 169 | | /// <see langword="false"/>. |
| | | 170 | | /// </returns> |
| | 0 | 171 | | public static bool operator !=(RunePosition left, RunePosition right) => !(left == right); |
| | | 172 | | |
| | | 173 | | /// <summary> |
| | | 174 | | /// An enumerator for retrieving <see cref="RunePosition"/> instances from Unicode data. |
| | | 175 | | /// </summary> |
| | | 176 | | /// <remarks> |
| | | 177 | | /// Methods are pattern-matched by compiler to allow using foreach pattern. |
| | | 178 | | /// </remarks> |
| | | 179 | | public ref struct Utf16Enumerator : IEnumerator<RunePosition> |
| | | 180 | | { |
| | | 181 | | private ReadOnlySpan<char> _original; |
| | | 182 | | private ReadOnlySpan<char> _remaining; |
| | | 183 | | |
| | | 184 | | /// <summary> |
| | | 185 | | /// The current <see cref="RunePosition"/> in the Unicode data. |
| | | 186 | | /// </summary> |
| | 0 | 187 | | public RunePosition Current { get; private set; } |
| | | 188 | | |
| | | 189 | | /// <summary> |
| | | 190 | | /// Returns the current enumerator instance. |
| | | 191 | | /// </summary> |
| | | 192 | | /// <returns>The current enumerator instance.</returns> |
| | 0 | 193 | | public Utf16Enumerator GetEnumerator() => this; |
| | | 194 | | |
| | | 195 | | internal Utf16Enumerator(ReadOnlySpan<char> buffer) |
| | | 196 | | { |
| | 0 | 197 | | _original = _remaining = buffer; |
| | 0 | 198 | | Current = default; |
| | 0 | 199 | | } |
| | | 200 | | |
| | | 201 | | /// <summary> |
| | | 202 | | /// Moves to the next <see cref="RunePosition"/> in the Unicode data. |
| | | 203 | | /// </summary> |
| | | 204 | | /// <returns> |
| | | 205 | | /// <see langword="true"/> if the enumerator was successfully advanced to the next <see cref="RunePosition"/>; |
| | | 206 | | /// <br /> |
| | | 207 | | /// <see langword="false"/> if the enumerator has passed the end of the span.</returns> |
| | | 208 | | public bool MoveNext() |
| | | 209 | | { |
| | 0 | 210 | | if (_remaining.IsEmpty) |
| | | 211 | | { |
| | | 212 | | // reached the end of the buffer |
| | 0 | 213 | | Current = default; |
| | 0 | 214 | | return false; |
| | | 215 | | } |
| | | 216 | | |
| | | 217 | | // In UTF-16 specifically, invalid sequences always have length 1, which is the same |
| | | 218 | | // length as the replacement character U+FFFD. This means that we can always bump the |
| | | 219 | | // next index by the current scalar's UTF-16 sequence length. This optimization is not |
| | | 220 | | // generally applicable; for example, enumerating scalars from UTF-8 cannot utilize |
| | | 221 | | // this same trick. |
| | | 222 | | |
| | 0 | 223 | | int scalarValue = Rune.ReadFirstRuneFromUtf16Buffer(_remaining); |
| | 0 | 224 | | if (scalarValue >= 0) |
| | | 225 | | { |
| | 0 | 226 | | Rune rune = Rune.UnsafeCreate((uint)scalarValue); |
| | 0 | 227 | | int length = rune.Utf16SequenceLength; |
| | 0 | 228 | | Current = new RunePosition(rune, Current.StartIndex + Current.Length, length, false); |
| | 0 | 229 | | _remaining = _remaining.Slice(length); |
| | | 230 | | } |
| | | 231 | | else |
| | | 232 | | { |
| | 0 | 233 | | Current = new RunePosition(Rune.ReplacementChar, Current.StartIndex + Current.Length, 1, true); |
| | 0 | 234 | | _remaining = _remaining.Slice(1); |
| | | 235 | | } |
| | 0 | 236 | | return true; |
| | | 237 | | } |
| | | 238 | | |
| | | 239 | | public void Reset() |
| | | 240 | | { |
| | 0 | 241 | | _remaining = _original; |
| | 0 | 242 | | Current = default; |
| | 0 | 243 | | } |
| | | 244 | | |
| | 0 | 245 | | object IEnumerator.Current => Current; |
| | | 246 | | |
| | 0 | 247 | | void IEnumerator.Reset() => Reset(); |
| | | 248 | | |
| | 0 | 249 | | void IDisposable.Dispose() { } |
| | | 250 | | } |
| | | 251 | | |
| | | 252 | | /// <summary> |
| | | 253 | | /// An enumerator for retrieving <see cref="RunePosition"/> instances from Unicode data. |
| | | 254 | | /// </summary> |
| | | 255 | | /// <remarks> |
| | | 256 | | /// Methods are pattern-matched by compiler to allow using foreach pattern. |
| | | 257 | | /// </remarks> |
| | | 258 | | public ref struct Utf8Enumerator : IEnumerator<RunePosition> |
| | | 259 | | { |
| | | 260 | | private ReadOnlySpan<byte> _original; |
| | | 261 | | private ReadOnlySpan<byte> _remaining; |
| | | 262 | | |
| | | 263 | | /// <summary> |
| | | 264 | | /// The current <see cref="RunePosition"/> in the Unicode data. |
| | | 265 | | /// </summary> |
| | 0 | 266 | | public RunePosition Current { get; private set; } |
| | | 267 | | |
| | | 268 | | /// <summary> |
| | | 269 | | /// Returns the current enumerator instance. |
| | | 270 | | /// </summary> |
| | | 271 | | /// <returns>The current enumerator instance.</returns> |
| | 0 | 272 | | public Utf8Enumerator GetEnumerator() => this; |
| | | 273 | | |
| | | 274 | | /// <summary> |
| | | 275 | | /// Initializes a new instance of the <see cref="Utf8Enumerator"/> struct. |
| | | 276 | | /// </summary> |
| | | 277 | | /// <param name="buffer">The buffer containing the Unicode data.</param> |
| | | 278 | | internal Utf8Enumerator(ReadOnlySpan<byte> buffer) |
| | | 279 | | { |
| | 0 | 280 | | _original = _remaining = buffer; |
| | 0 | 281 | | Current = default; |
| | 0 | 282 | | } |
| | | 283 | | |
| | | 284 | | /// <summary> |
| | | 285 | | /// Moves to the next <see cref="RunePosition"/> in the Unicode data. |
| | | 286 | | /// </summary> |
| | | 287 | | /// <returns> |
| | | 288 | | /// <see langword="true"/> if the enumerator was successfully advanced to the next <see cref="RunePosition"/>; |
| | | 289 | | /// <br /> |
| | | 290 | | /// <see langword="false"/> if the enumerator has passed the end of the span. |
| | | 291 | | /// </returns> |
| | | 292 | | public bool MoveNext() |
| | | 293 | | { |
| | 0 | 294 | | if (_remaining.IsEmpty) |
| | | 295 | | { |
| | | 296 | | // reached the end of the buffer |
| | 0 | 297 | | Current = default; |
| | 0 | 298 | | return false; |
| | | 299 | | } |
| | | 300 | | |
| | 0 | 301 | | bool wasReplaced = Rune.DecodeFromUtf8(_remaining, out Rune rune, out int charsConsumed) != OperationStatus. |
| | 0 | 302 | | Current = new RunePosition(rune, Current.StartIndex + Current.Length, charsConsumed, wasReplaced); |
| | 0 | 303 | | _remaining = _remaining.Slice(charsConsumed); |
| | 0 | 304 | | return true; |
| | | 305 | | } |
| | | 306 | | |
| | | 307 | | public void Reset() |
| | | 308 | | { |
| | 0 | 309 | | _remaining = _original; |
| | 0 | 310 | | Current = default; |
| | 0 | 311 | | } |
| | | 312 | | |
| | 0 | 313 | | object IEnumerator.Current => Current; |
| | | 314 | | |
| | 0 | 315 | | void IEnumerator.Reset() => Reset(); |
| | | 316 | | |
| | 0 | 317 | | void IDisposable.Dispose() { } |
| | | 318 | | } |
| | | 319 | | } |
| | | 320 | | |