fix(serialization): potential infinite loop when inlining references

Author: baywet

src/Microsoft.OpenApi/Extensions/OpenAPIWriterExtensions.cs

@@ -15,5 +15,13 @@ internal static OpenApiWriterSettings GetSettings(this IOpenApiWriter openApiWri
             }
             return new();
         }
+
+        internal static void CountInlineReference(this IOpenApiWriter openApiWriter)
+        {
+            if (openApiWriter is OpenApiWriterBase @base)
+            {
+                @base.CountInlineReference();
+            }
+        }
     }
 }

src/Microsoft.OpenApi/Models/References/BaseOpenApiReferenceHolder.cs

@@ -207,7 +207,7 @@ protected BaseOpenApiReferenceHolder(string referenceId, OpenApiDocument? hostDo
     /// <inheritdoc/>
     public virtual void SerializeAsV3(IOpenApiWriter writer)
     {
-        if (!writer.GetSettings().ShouldInlineReference(Reference) 
+        if (!writer.GetSettings().ShouldInlineReference(Reference)
             || Reference.Type == ReferenceType.Tag) // tags are held as references need to drop in.
         {
             Reference.SerializeAsV3(writer);
@@ -267,6 +267,7 @@ private protected void SerializeInternal(IOpenApiWriter writer,
         Action<IOpenApiWriter, U> action)
     {
         Utils.CheckArgumentNull(writer);
+        writer.CountInlineReference();
         ApplyToTarget(element => action(writer, element));
     }
 }

src/Microsoft.OpenApi/Models/References/OpenApiSchemaReference.cs

@@ -256,8 +256,14 @@ private void SerializeAsWithoutLoops(IOpenApiWriter writer, Action<IOpenApiWrite
             }
             else
             {
-                SerializeInternal(writer, (w, element) => action(w, element));
-                writer.GetSettings().LoopDetector.PopLoop<IOpenApiSchema>();
+                try
+                {
+                    SerializeInternal(writer, (w, element) => action(w, element));
+                }
+                finally
+                {
+                    writer.GetSettings().LoopDetector.PopLoop<IOpenApiSchema>();
+                }
             }
 
         }

src/Microsoft.OpenApi/Properties/SRResource.Designer.cs

@@ -132,6 +132,9 @@
   <data name="IndentationLevelInvalid" xml:space="preserve">
     <value>Indentation level cannot be lower than 0.</value>
   </data>
+  <data name="MaximumInlineReferenceCountExceeded" xml:space="preserve">
+    <value>The maximum number of inline references ({0}) has been exceeded.</value>
+  </data>
   <data name="InputItemShouldBeType" xml:space="preserve">
     <value>The input item should be in type of '{0}'.</value>
   </data>

src/Microsoft.OpenApi/Properties/SRResource.resx

@@ -1,2 +1,4 @@
 #nullable enable
 Microsoft.OpenApi.OpenApiDocument.GetOperationById(string! operationId) -> Microsoft.OpenApi.OpenApiOperation?
+Microsoft.OpenApi.OpenApiWriterSettings.MaxInlineReferenceCount.get -> int
+Microsoft.OpenApi.OpenApiWriterSettings.MaxInlineReferenceCount.set -> void

src/Microsoft.OpenApi/PublicAPI.Unshipped.txt

@@ -35,6 +35,7 @@ public abstract class OpenApiWriterBase : IOpenApiWriter
         /// Number which specifies the level of indentation.
         /// </summary>
         private int _indentLevel;
+        private int _inlineReferenceCount;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="OpenApiWriterBase"/> class.
@@ -70,6 +71,20 @@ protected OpenApiWriterBase(TextWriter textWriter, OpenApiWriterSettings? settin
         /// </summary>
         protected TextWriter Writer { get; }
 
+        internal void CountInlineReference()
+        {
+            if (_inlineReferenceCount >= Settings.MaxInlineReferenceCount)
+            {
+                throw new OpenApiWriterException(
+                    string.Format(
+                        CultureInfo.InvariantCulture,
+                        SRResource.MaximumInlineReferenceCountExceeded,
+                        Settings.MaxInlineReferenceCount));
+            }
+
+            _inlineReferenceCount++;
+        }
+
         /// <summary>
         /// Write start object.
         /// </summary>
@@ -319,7 +334,7 @@ public virtual void WriteIndentation()
                 Writer.Write(IndentationString);
             }
         }
-        
+
         /// <summary>
         /// Get current scope.
         /// </summary>

src/Microsoft.OpenApi/Writers/OpenApiWriterBase.cs

@@ -5,7 +5,10 @@
     /// </summary>
     public class OpenApiWriterSettings
     {
+        private const int DefaultMaxInlineReferenceCount = 10_000;
+
         internal LoopDetector LoopDetector { get; } = new();
+
         /// <summary>
         /// Indicates if local references should be rendered as an inline object
         /// </summary>
@@ -16,6 +19,12 @@ public class OpenApiWriterSettings
         /// </summary>
         public bool InlineExternalReferences { get; set; }
 
+        /// <summary>
+        /// Gets or sets the maximum number of references that can be expanded inline by a writer.
+        /// </summary>
+        /// <remarks>The default is 10,000.</remarks>
+        public int MaxInlineReferenceCount { get; set; } = DefaultMaxInlineReferenceCount;
+
         internal bool ShouldInlineReference(BaseOpenApiReference reference)
         {
             return (reference.IsLocal && InlineLocalReferences)

src/Microsoft.OpenApi/Writers/OpenApiWriterSettings.cs

@@ -0,0 +1,220 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using System;
+using System.Globalization;
+using System.IO;
+using System.Text;
+using Xunit;
+
+namespace Microsoft.OpenApi.Tests.Writers
+{
+    public class OpenApiWriterInlineReferenceTests
+    {
+        private const int InlineReferenceLimit = 128;
+        private const int OutputCharacterLimit = 100_000;
+
+        [Theory]
+        [InlineData(OpenApiSpecVersion.OpenApi2_0, false)]
+        [InlineData(OpenApiSpecVersion.OpenApi2_0, true)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_0, false)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_0, true)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_1, false)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_1, true)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_2, false)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_2, true)]
+        public void SerializeWithRepeatedInlineReferencesStopsBeforeWriterLimit(
+            OpenApiSpecVersion specVersion,
+            bool useYaml)
+        {
+            var readResult = OpenApiDocument.Parse(CreateDocumentWithRepeatedReferences(18), "json");
+            Assert.Empty(readResult.Diagnostic.Errors);
+            Assert.NotNull(readResult.Document);
+
+            using var textWriter = new CharacterLimitedTextWriter(OutputCharacterLimit);
+            IOpenApiWriter writer = useYaml
+                ? new OpenApiYamlWriter(textWriter, CreateWriterSettings(InlineReferenceLimit))
+                : new OpenApiJsonWriter(textWriter, CreateWriterSettings(InlineReferenceLimit));
+
+            var exception = Assert.Throws<OpenApiWriterException>(
+                () => Serialize(readResult.Document, writer, specVersion));
+            Assert.Contains(InlineReferenceLimit.ToString(CultureInfo.InvariantCulture), exception.Message);
+        }
+
+        [Theory]
+        [InlineData(OpenApiSpecVersion.OpenApi2_0)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_0)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_1)]
+        [InlineData(OpenApiSpecVersion.OpenApi3_2)]
+        public void SerializeWithInlineReferencesBelowLimitWorks(OpenApiSpecVersion specVersion)
+        {
+            var readResult = OpenApiDocument.Parse(CreateDocumentWithRepeatedReferences(4), "json");
+            Assert.Empty(readResult.Diagnostic.Errors);
+            Assert.NotNull(readResult.Document);
+
+            using var textWriter = new StringWriter(CultureInfo.InvariantCulture);
+            var writer = new OpenApiJsonWriter(textWriter, CreateWriterSettings(32));
+
+            Serialize(readResult.Document, writer, specVersion);
+
+            Assert.NotEmpty(textWriter.ToString());
+        }
+
+        [Fact]
+        public void ReusingSettingsWithNewWriterStartsANewInlineReferenceCount()
+        {
+            var largeReadResult = OpenApiDocument.Parse(CreateDocumentWithRepeatedReferences(6), "json");
+            Assert.Empty(largeReadResult.Diagnostic.Errors);
+            Assert.NotNull(largeReadResult.Document);
+            var smallReadResult = OpenApiDocument.Parse(CreateDocumentWithRepeatedReferences(4), "json");
+            Assert.Empty(smallReadResult.Diagnostic.Errors);
+            Assert.NotNull(smallReadResult.Document);
+            var settings = CreateWriterSettings(32);
+
+            using var firstOutput = new StringWriter(CultureInfo.InvariantCulture);
+            Assert.Throws<OpenApiWriterException>(
+                () => Serialize(
+                    largeReadResult.Document,
+                    new OpenApiJsonWriter(firstOutput, settings),
+                    OpenApiSpecVersion.OpenApi3_0));
+
+            using var secondOutput = new StringWriter(CultureInfo.InvariantCulture);
+            Serialize(
+                smallReadResult.Document,
+                new OpenApiJsonWriter(secondOutput, settings),
+                OpenApiSpecVersion.OpenApi3_0);
+
+            Assert.DoesNotContain("$ref", secondOutput.ToString());
+        }
+
+        private static string CreateDocumentWithRepeatedReferences(int levels)
+        {
+            var schemas = new StringBuilder();
+            for (var i = 0; i < levels; i++)
+            {
+                if (i > 0)
+                {
+                    schemas.Append(',');
+                }
+
+                schemas
+                    .Append("\"s")
+                    .Append(i.ToString(CultureInfo.InvariantCulture))
+                    .Append("\":{\"allOf\":[{\"$ref\":\"#/components/schemas/s")
+                    .Append((i + 1).ToString(CultureInfo.InvariantCulture))
+                    .Append("\"},{\"$ref\":\"#/components/schemas/s")
+                    .Append((i + 1).ToString(CultureInfo.InvariantCulture))
+                    .Append("\"}]}");
+            }
+
+            schemas
+                .Append(",\"s")
+                .Append(levels.ToString(CultureInfo.InvariantCulture))
+                .Append("\":{\"type\":\"string\"}");
+
+            return $$"""
+                {
+                  "openapi": "3.0.0",
+                  "info": {
+                    "title": "Repeated references",
+                    "version": "1.0.0"
+                  },
+                  "paths": {
+                    "/test": {
+                      "get": {
+                        "responses": {
+                          "200": {
+                            "description": "OK",
+                            "content": {
+                              "application/json": {
+                                "schema": {
+                                  "$ref": "#/components/schemas/s0"
+                                }
+                              }
+                            }
+                          }
+                        }
+                      }
+                    }
+                  },
+                  "components": {
+                    "schemas": {
+                      {{schemas}}
+                    }
+                  }
+                }
+                """;
+        }
+
+        private static void Serialize(
+            OpenApiDocument document,
+            IOpenApiWriter writer,
+            OpenApiSpecVersion specVersion)
+        {
+            switch (specVersion)
+            {
+                case OpenApiSpecVersion.OpenApi2_0:
+                    document.SerializeAsV2(writer);
+                    break;
+                case OpenApiSpecVersion.OpenApi3_0:
+                    document.SerializeAsV3(writer);
+                    break;
+                case OpenApiSpecVersion.OpenApi3_1:
+                    document.SerializeAsV31(writer);
+                    break;
+                case OpenApiSpecVersion.OpenApi3_2:
+                    document.SerializeAsV32(writer);
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(specVersion), specVersion, null);
+            }
+        }
+
+        private static OpenApiWriterSettings CreateWriterSettings(int maxInlineReferenceCount)
+        {
+            return new OpenApiWriterSettings
+            {
+                InlineLocalReferences = true,
+                MaxInlineReferenceCount = maxInlineReferenceCount
+            };
+        }
+
+        private sealed class CharacterLimitedTextWriter(int characterLimit) : TextWriter
+        {
+            private int _characterCount;
+
+            public override Encoding Encoding => Encoding.UTF8;
+
+            public override void Write(char value)
+            {
+                CountCharacters(1);
+            }
+
+            public override void Write(string value)
+            {
+                CountCharacters(value?.Length ?? 0);
+            }
+
+            public override void Write(char[] buffer, int index, int count)
+            {
+                CountCharacters(count);
+            }
+
+            public override void Write(ReadOnlySpan<char> buffer)
+            {
+                CountCharacters(buffer.Length);
+            }
+
+            private void CountCharacters(int count)
+            {
+                _characterCount += count;
+                if (_characterCount > characterLimit)
+                {
+                    throw new CharacterLimitExceededException();
+                }
+            }
+        }
+
+        private sealed class CharacterLimitExceededException : Exception;
+    }
+}