| | | 1 | | // Licensed to the .NET Foundation under one or more agreements. |
| | | 2 | | // The .NET Foundation licenses this file to you under the MIT license. |
| | | 3 | | |
| | | 4 | | using System.Buffers; |
| | | 5 | | using System.Text.Unicode; |
| | | 6 | | using System.Text; |
| | | 7 | | |
| | | 8 | | namespace DotnetFuzzing.Fuzzers; |
| | | 9 | | |
| | | 10 | | // Adapted from https://github.com/GrabYourPitchforks/utf8fuzz |
| | | 11 | | internal sealed class UTF8Fuzzer : IFuzzer |
| | | 12 | | { |
| | 0 | 13 | | public string[] TargetAssemblies => []; |
| | 1 | 14 | | public string[] TargetCoreLibPrefixes { get; } = ["System.Text"]; |
| | | 15 | | |
| | 0 | 16 | | private static readonly Encoding s_encodingWithCustomReplacement = |
| | 0 | 17 | | Encoding.GetEncoding("utf-8", EncoderFallback.ExceptionFallback, new DecoderReplacementFallback("{BAD}")); |
| | | 18 | | |
| | 0 | 19 | | private static readonly ArrayBufferWriter<char> s_replacementBufferWriter = new(4096); |
| | | 20 | | |
| | | 21 | | public void FuzzTarget(ReadOnlySpan<byte> bytes) |
| | 0 | 22 | | { |
| | 0 | 23 | | using var poisonAfter = PooledBoundedMemory<byte>.Rent(bytes, PoisonPagePlacement.After); |
| | | 24 | | |
| | 0 | 25 | | Test(poisonAfter.Span); |
| | 0 | 26 | | } |
| | | 27 | | |
| | | 28 | | private static void Test(ReadOnlySpan<byte> input) |
| | 0 | 29 | | { |
| | 0 | 30 | | int charCount = Encoding.UTF8.GetCharCount(input); |
| | | 31 | | |
| | 0 | 32 | | Assert.Equal(charCount, GetCharCountFromRunes(input)); |
| | | 33 | | |
| | 0 | 34 | | using var chars = PooledBoundedMemory<char>.Rent(charCount, PoisonPagePlacement.After); |
| | 0 | 35 | | using var chars2 = PooledBoundedMemory<char>.Rent(charCount + 1024, PoisonPagePlacement.After); |
| | | 36 | | |
| | 0 | 37 | | Assert.Equal(charCount, Encoding.UTF8.GetChars(input, chars.Span)); |
| | | 38 | | |
| | 0 | 39 | | CompareUtf8AndUtf16RuneEnumeration(input, chars.Span); |
| | | 40 | | |
| | | 41 | | // ToUtf16 with replace=true and exact size buffer. |
| | 0 | 42 | | { |
| | 0 | 43 | | OperationStatus opStatus = Utf8.ToUtf16(input, chars2.Span.Slice(0, charCount), out int bytesReadJustNow, ou |
| | | 44 | | |
| | 0 | 45 | | Assert.Equal(OperationStatus.Done, opStatus); |
| | 0 | 46 | | Assert.Equal(input.Length, bytesReadJustNow); |
| | 0 | 47 | | Assert.Equal(charCount, charsWrittenJustNow); |
| | 0 | 48 | | Assert.SequenceEqual<char>(chars.Span, chars2.Span.Slice(0, charCount)); |
| | 0 | 49 | | } |
| | | 50 | | |
| | | 51 | | // ToUtf16 with replace=true and extra large buffer. |
| | 0 | 52 | | { |
| | 0 | 53 | | OperationStatus opStatus = Utf8.ToUtf16(input, chars2.Span, out int bytesReadJustNow, out int charsWrittenJu |
| | | 54 | | |
| | 0 | 55 | | Assert.Equal(OperationStatus.Done, opStatus); |
| | 0 | 56 | | Assert.Equal(input.Length, bytesReadJustNow); |
| | 0 | 57 | | Assert.Equal(charCount, charsWrittenJustNow); |
| | 0 | 58 | | Assert.SequenceEqual<char>(chars.Span, chars2.Span.Slice(0, charCount)); |
| | 0 | 59 | | } |
| | | 60 | | |
| | | 61 | | // Running ToUtf16 with replace=false and extra large buffer. |
| | 0 | 62 | | { |
| | 0 | 63 | | ReadOnlySpan<byte> utf8 = input; |
| | 0 | 64 | | Span<char> output = chars2.Span; |
| | 0 | 65 | | output.Clear(); |
| | | 66 | | |
| | 0 | 67 | | while (!utf8.IsEmpty) |
| | 0 | 68 | | { |
| | 0 | 69 | | OperationStatus opStatus = Utf8.ToUtf16(utf8, output, out int bytesReadJustNow, out int charsWrittenJust |
| | | 70 | | |
| | 0 | 71 | | CompareUtf8AndUtf16RuneEnumeration(utf8.Slice(0, bytesReadJustNow), output.Slice(0, charsWrittenJustNow) |
| | | 72 | | |
| | 0 | 73 | | utf8 = utf8.Slice(bytesReadJustNow); |
| | | 74 | | |
| | 0 | 75 | | if (opStatus != OperationStatus.Done) |
| | 0 | 76 | | { |
| | 0 | 77 | | Assert.Equal(OperationStatus.InvalidData, opStatus); |
| | | 78 | | |
| | | 79 | | // Skip over invalid data |
| | 0 | 80 | | Rune.DecodeFromUtf8(utf8, out _, out int bytesToSkip); |
| | 0 | 81 | | utf8 = utf8.Slice(bytesToSkip); |
| | 0 | 82 | | } |
| | 0 | 83 | | } |
| | 0 | 84 | | } |
| | | 85 | | |
| | | 86 | | // Trying a custom decoder replacement. |
| | 0 | 87 | | { |
| | 0 | 88 | | ReadOnlySpan<byte> utf8 = input; |
| | | 89 | | |
| | 0 | 90 | | string decoded = s_encodingWithCustomReplacement.GetString(utf8); |
| | | 91 | | |
| | 0 | 92 | | using var decodedChars = PooledBoundedMemory<char>.Rent(decoded.Length, PoisonPagePlacement.After); |
| | | 93 | | |
| | 0 | 94 | | s_replacementBufferWriter.ResetWrittenCount(); |
| | | 95 | | |
| | 0 | 96 | | while (!utf8.IsEmpty) |
| | 0 | 97 | | { |
| | 0 | 98 | | OperationStatus opStatus = Utf8.ToUtf16(utf8, decodedChars.Span, out int bytesReadJustNow, out int chars |
| | 0 | 99 | | s_replacementBufferWriter.Write(decodedChars.Span.Slice(0, charsWrittenJustNow)); |
| | | 100 | | |
| | 0 | 101 | | utf8 = utf8.Slice(bytesReadJustNow); |
| | | 102 | | |
| | 0 | 103 | | if (opStatus != OperationStatus.Done) |
| | 0 | 104 | | { |
| | 0 | 105 | | Assert.Equal(OperationStatus.InvalidData, opStatus); |
| | | 106 | | |
| | | 107 | | // Skip over invalid data |
| | 0 | 108 | | Rune.DecodeFromUtf8(utf8, out _, out int bytesToSkip); |
| | 0 | 109 | | utf8 = utf8.Slice(bytesToSkip); |
| | | 110 | | |
| | 0 | 111 | | s_replacementBufferWriter.Write("{BAD}"); |
| | 0 | 112 | | } |
| | 0 | 113 | | } |
| | | 114 | | |
| | 0 | 115 | | Assert.SequenceEqual(decoded, s_replacementBufferWriter.WrittenSpan); |
| | 0 | 116 | | } |
| | 0 | 117 | | } |
| | | 118 | | |
| | | 119 | | private static int GetCharCountFromRunes(ReadOnlySpan<byte> utf8) |
| | 0 | 120 | | { |
| | 0 | 121 | | int charCount = 0; |
| | | 122 | | |
| | 0 | 123 | | while (!utf8.IsEmpty) |
| | 0 | 124 | | { |
| | 0 | 125 | | Rune.DecodeFromUtf8(utf8, out Rune thisRune, out int bytesConsumed); |
| | 0 | 126 | | charCount += thisRune.Utf16SequenceLength; // ok if U+FFFD replacement |
| | 0 | 127 | | utf8 = utf8.Slice(bytesConsumed); |
| | 0 | 128 | | } |
| | | 129 | | |
| | 0 | 130 | | return charCount; |
| | 0 | 131 | | } |
| | | 132 | | |
| | | 133 | | private static void CompareUtf8AndUtf16RuneEnumeration(ReadOnlySpan<byte> utf8, ReadOnlySpan<char> utf16, bool expec |
| | 0 | 134 | | { |
| | 0 | 135 | | while (!utf8.IsEmpty && !utf16.IsEmpty) |
| | 0 | 136 | | { |
| | 0 | 137 | | OperationStatus utf8Status = Rune.DecodeFromUtf8(utf8, out Rune inputUtf8Rune, out int bytesConsumed); |
| | 0 | 138 | | OperationStatus utf16Status = Rune.DecodeFromUtf16(utf16, out Rune inputUtf16Rune, out int charsConsumed); |
| | | 139 | | |
| | 0 | 140 | | if (expectIsValid) |
| | 0 | 141 | | { |
| | 0 | 142 | | Assert.Equal(OperationStatus.Done, utf8Status); |
| | 0 | 143 | | Assert.Equal(OperationStatus.Done, utf16Status); |
| | 0 | 144 | | } |
| | | 145 | | |
| | 0 | 146 | | Assert.Equal(inputUtf8Rune, inputUtf16Rune); |
| | | 147 | | |
| | 0 | 148 | | utf8 = utf8.Slice(bytesConsumed); |
| | 0 | 149 | | utf16 = utf16.Slice(charsConsumed); |
| | 0 | 150 | | } |
| | | 151 | | |
| | 0 | 152 | | Assert.Equal(utf8.Length, utf16.Length); |
| | 0 | 153 | | } |
| | | 154 | | } |