fix!: resolve local references outside the components, in both versions (#63)

* fix!: resolve local references outside the components, in both versions

Local references that do not point at a registered component, such as into
another channel, operation or message, are resolved from the document. That
fallback crashed or failed for most of them:

- 2.x documents were resolved against their 3.x re-serialization, so 2.x
  pointers into publish/subscribe and channel parameters never resolved,
  and 2.x payloads lost their schemaFormat. They now resolve against the
  original document, and the payload's schemaFormat is kept in scoped temp
  storage for the 2.x version service to apply.
- Correlation ID and operation trait fragments had no loader, and server
  variable, tag, external docs, reply and reply address references had no
  fallback, so reading threw or reported an error.
- The walker skipped server variable and operation references.
- A 2.x channel referencing another root channel crashed while registering
  components. It now points at the same channel, and is written with that
  channel's address in 2.x.

Writing kept references whose target the written version does not have,
such as a 2.x "#/channels/a/publish/message" written as 3.x, or a 3.x
"#/operations/op/traits/0" written as 2.x. Those are now written inline; a
reference that resolves in the written document stays a reference. The
writer also sees what the reader resolved outside the components, and 2.x
writes referenced tags and external docs inline instead of dropping them.

BREAKING CHANGE: JsonPointerExtensions is removed; use JsonPointer.Find.

* fix: inline a 2.x channel reference whose target is not written

A 2.x channel reference is written with its target's address, since 2.x
keys channels by address. When the target is not one of the document's
channels, for example because it was removed after reading, that address
points nowhere and did not match the reference, so it was left dangling.
Such a reference now keeps its own pointer, which is found dangling and
written inline.

Author: VisualBean

src/ByteBard.AsyncAPI.Readers/AsyncApiJsonDocumentReader.cs

@@ -11,7 +11,6 @@ namespace ByteBard.AsyncAPI.Readers
     using ByteBard.AsyncAPI.Models;
     using ByteBard.AsyncAPI.Models.Interfaces;
     using ByteBard.AsyncAPI.Readers.Interface;
-    using ByteBard.AsyncAPI.Readers.ParseNodes;
     using ByteBard.AsyncAPI.Readers.Services;
     using ByteBard.AsyncAPI.Services;
     using ByteBard.AsyncAPI.Validations;
@@ -381,17 +380,17 @@ private IAsyncApiSerializable ResolveStreamReference(Stream stream, IAsyncApiRef
                 }
 
                 var pointerResult = pointer.Find(json);
-                if (pointerResult != null)
-                {
-                    json = pointerResult;
-                }
-                else
+                if (pointerResult == null)
                 {
                     diagnostic.Errors.Add(new AsyncApiError(reference.Reference.Reference, "Could not resolve reference fragment."));
                     return null;
                 }
+
+                json = pointerResult;
             }
 
+            json = this.context.VersionService.PrepareReferenceFragment(json, reference, this.context);
+
             AsyncApiDiagnostic fragmentDiagnostic = new AsyncApiDiagnostic
             {
                 SpecificationVersion = diagnostic.SpecificationVersion,
@@ -453,6 +452,21 @@ private IAsyncApiSerializable ResolveStreamReference(Stream stream, IAsyncApiRef
                 case ReferenceType.MessageBindings:
                     result = this.ReadFragment<AsyncApiBindings<IMessageBinding>>(json, diagnostic.SpecificationVersion, out fragmentDiagnostic);
                     break;
+                case ReferenceType.ServerVariable:
+                    result = this.ReadFragment<AsyncApiServerVariable>(json, diagnostic.SpecificationVersion, out fragmentDiagnostic);
+                    break;
+                case ReferenceType.Tag:
+                    result = this.ReadFragment<AsyncApiTag>(json, diagnostic.SpecificationVersion, out fragmentDiagnostic);
+                    break;
+                case ReferenceType.ExternalDocs:
+                    result = this.ReadFragment<AsyncApiExternalDocumentation>(json, diagnostic.SpecificationVersion, out fragmentDiagnostic);
+                    break;
+                case ReferenceType.OperationReply when diagnostic.SpecificationVersion == AsyncApiVersion.AsyncApi3_0:
+                    result = this.ReadFragment<AsyncApiOperationReply>(json, diagnostic.SpecificationVersion, out fragmentDiagnostic);
+                    break;
+                case ReferenceType.OperationReplyAddress when diagnostic.SpecificationVersion == AsyncApiVersion.AsyncApi3_0:
+                    result = this.ReadFragment<AsyncApiOperationReplyAddress>(json, diagnostic.SpecificationVersion, out fragmentDiagnostic);
+                    break;
                 default:
                     diagnostic.Errors.Add(new AsyncApiError(reference.Reference.Reference, "Could not resolve reference."));
                     break;

src/ByteBard.AsyncAPI.Readers/Interface/IAsyncApiVersionService.cs

@@ -1,5 +1,6 @@
 namespace ByteBard.AsyncAPI.Readers.Interface
 {
+    using System.Text.Json.Nodes;
     using ByteBard.AsyncAPI.Models;
     using ByteBard.AsyncAPI.Models.Interfaces;
     using ByteBard.AsyncAPI.Readers.ParseNodes;
@@ -12,5 +13,10 @@ T LoadElement<T>(ParseNode node)
             where T : IAsyncApiElement;
 
         AsyncApiDocument LoadDocument(RootNode rootNode);
+
+        /// <summary>
+        /// Adjusts the fragment a reference points at before it is loaded, for what the version keeps elsewhere.
+        /// </summary>
+        JsonNode PrepareReferenceFragment(JsonNode fragment, IAsyncApiReferenceable reference, ParsingContext context);
     }
-}
+}

src/ByteBard.AsyncAPI.Readers/ParseNodes/JsonPointerExtensions.cs

@@ -90,7 +90,7 @@ internal AsyncApiDocument Parse(JsonNode jsonNode)
 
                     this.Workspace.SetRootDocument(doc);
                     this.Workspace.RegisterComponents(doc);
-                    this.Workspace.RegisterComponent(string.Empty, this.ParseToStream(doc));
+                    this.Workspace.RegisterComponent(string.Empty, this.ParseToStream(jsonNode));
 
                     this.Diagnostic.SpecificationVersion = AsyncApiVersion.AsyncApi2_0;
                     break;

src/ByteBard.AsyncAPI.Readers/ParsingContext.cs

@@ -8,5 +8,6 @@ public static class TempStorageKeys
         public const string ComponentMessages = "ComponentMessages";
         public const string ChannelAddresses = "ChannelAddresses";
         public const string ParameterSchemaReferences = "ParameterSchemaReferences";
+        public const string PayloadSchemaFormat = "PayloadSchemaFormat";
     }
 }

src/ByteBard.AsyncAPI.Readers/TempStorageKeys.cs

@@ -36,7 +36,7 @@ public static AsyncApiChannel LoadChannel(ParseNode node, string channelAddress
             var pointer = mapNode.GetReferencePointer();
             if (pointer != null)
             {
-                return new AsyncApiChannelReference(pointer);
+                return new AsyncApiChannelReference(NormalizeChannelPointer(pointer));
             }
 
             var channel = new AsyncApiChannel();
@@ -53,6 +53,23 @@ public static AsyncApiChannel LoadChannel(ParseNode node, string channelAddress
             return channel;
         }
 
+        /// <summary>
+        /// 2.x keys channels by address, which is normalized into the channel's key, so a reference to a root channel
+        /// is pointed at that key.
+        /// </summary>
+        private static string NormalizeChannelPointer(string pointer)
+        {
+            const string channelsPrefix = "#/channels/";
+            if (!pointer.StartsWith(channelsPrefix)
+                || !JsonPointer.TryFromUriFragment(pointer.Substring(1), out var parsed)
+                || parsed.Tokens.Length != 2)
+            {
+                return pointer;
+            }
+
+            return channelsPrefix + JsonPointer.EscapeToken(NormalizeChannelKey(parsed.Tokens[1]));
+        }
+
         public static string NormalizeChannelKey(string channelKey, ParseNode node = null)
         {
             string newKey = string.Empty;

src/ByteBard.AsyncAPI.Readers/V2/AsyncApiChannelDeserializer.cs

@@ -152,7 +152,14 @@ public static AsyncApiMessage LoadMessage(ParseNode node)
                 var payload = LoadPayload(mapNode["payload"].Value, schema);
                 if (payload is IAsyncApiReferenceable reference && !reference.Reference.IsExternal)
                 {
-                    message.Payload = new AsyncApiMultiFormatSchemaReference(reference.Reference.Reference);
+                    var payloadReference = new AsyncApiMultiFormatSchemaReference(reference.Reference.Reference);
+                    message.Payload = payloadReference;
+
+                    // 2.x keeps the schema format on the message, so keep it for when the reference is resolved.
+                    if (schema != null)
+                    {
+                        node.Context.SetTempStorage(TempStorageKeys.PayloadSchemaFormat, schema, payloadReference);
+                    }
                 }
                 else
                 {

src/ByteBard.AsyncAPI.Readers/V2/AsyncApiMessageDeserializer.cs

@@ -1,7 +1,8 @@
-namespace ByteBard.AsyncAPI.Readers.V2
+namespace ByteBard.AsyncAPI.Readers.V2
 {
     using System;
     using System.Collections.Generic;
+    using System.Text.Json.Nodes;
     using ByteBard.AsyncAPI.Exceptions;
     using ByteBard.AsyncAPI.Models;
     using ByteBard.AsyncAPI.Models.Interfaces;
@@ -32,12 +33,11 @@ public AsyncApiV2VersionService(AsyncApiDiagnostic diagnostic)
             [typeof(AsyncApiOAuthFlows)] = AsyncApiV2Deserializer.LoadOAuthFlows,
             [typeof(AsyncApiOperation)] = AsyncApiV2Deserializer.LoadOperation,
             [typeof(AsyncApiParameter)] = AsyncApiV2Deserializer.LoadParameter,
-            [typeof(AsyncApiJsonSchema)] = AsyncApiJsonSchemaDeserializer.LoadSchema,
-            [typeof(AsyncApiAvroSchema)] = AsyncApiAvroSchemaDeserializer.LoadSchema,
             [typeof(AsyncApiJsonSchema)] = AsyncApiV2Deserializer.LoadJsonSchemaPayload,
             [typeof(AsyncApiAvroSchema)] = AsyncApiV2Deserializer.LoadAvroPayload,
-            // 2.x has no Multi Format Schema Object. Only the local reference fallback asks for one, and it reads the document after the upgrade to 3.0.
             [typeof(AsyncApiMultiFormatSchema)] = AsyncApiV3Deserializer.LoadMultiFormatSchema,
+            [typeof(AsyncApiCorrelationId)] = AsyncApiV2Deserializer.LoadCorrelationId,
+            [typeof(AsyncApiOperationTrait)] = AsyncApiV2Deserializer.LoadOperationTrait,
             [typeof(AsyncApiSecurityScheme)] = AsyncApiV2Deserializer.LoadSecurityScheme,
             [typeof(AsyncApiServer)] = AsyncApiV2Deserializer.LoadServer,
             [typeof(AsyncApiServerVariable)] = AsyncApiV2Deserializer.LoadServerVariable,
@@ -81,6 +81,25 @@ public AsyncApiDocument LoadDocument(RootNode rootNode)
             return AsyncApiV2Deserializer.LoadAsyncApi(rootNode);
         }
 
+        /// <summary>
+        /// 2.x keeps a payload's schema format on its message, which the reader stored with the payload reference.
+        /// Wrap the payload with it as a Multi Format Schema Object, so it is read in that format.
+        /// </summary>
+        public JsonNode PrepareReferenceFragment(JsonNode fragment, IAsyncApiReferenceable reference, ParsingContext context)
+        {
+            if (reference is not AsyncApiMultiFormatSchemaReference
+                || context.GetFromTempStorage<string>(TempStorageKeys.PayloadSchemaFormat, reference) is not string schemaFormat)
+            {
+                return fragment;
+            }
+
+            return new JsonObject
+            {
+                ["schemaFormat"] = schemaFormat,
+                ["schema"] = fragment.DeepClone(),
+            };
+        }
+
         public T LoadElement<T>(ParseNode node)
             where T : IAsyncApiElement
         {

src/ByteBard.AsyncAPI.Readers/V2/AsyncApiV2VersionService.cs

@@ -2,6 +2,7 @@
 {
     using System;
     using System.Collections.Generic;
+    using System.Text.Json.Nodes;
     using ByteBard.AsyncAPI.Exceptions;
     using ByteBard.AsyncAPI.Models;
     using ByteBard.AsyncAPI.Models.Interfaces;
@@ -38,6 +39,8 @@ public AsyncApiV3VersionService(AsyncApiDiagnostic diagnostic)
             [typeof(AsyncApiAvroSchema)] = AsyncApiAvroSchemaDeserializer.LoadSchema,
             [typeof(AsyncApiSecurityScheme)] = AsyncApiV3Deserializer.LoadSecurityScheme,
             [typeof(AsyncApiMultiFormatSchema)] = AsyncApiV3Deserializer.LoadMultiFormatSchema,
+            [typeof(AsyncApiCorrelationId)] = AsyncApiV3Deserializer.LoadCorrelationId,
+            [typeof(AsyncApiOperationTrait)] = AsyncApiV3Deserializer.LoadOperationTrait,
             [typeof(AsyncApiServer)] = AsyncApiV3Deserializer.LoadServer,
             [typeof(AsyncApiServerVariable)] = AsyncApiV3Deserializer.LoadServerVariable,
             [typeof(AsyncApiTag)] = AsyncApiV3Deserializer.LoadTag,
@@ -80,6 +83,11 @@ public AsyncApiDocument LoadDocument(RootNode rootNode)
             return AsyncApiV3Deserializer.LoadAsyncApi(rootNode);
         }
 
+        public JsonNode PrepareReferenceFragment(JsonNode fragment, IAsyncApiReferenceable reference, ParsingContext context)
+        {
+            return fragment;
+        }
+
         public T LoadElement<T>(ParseNode node)
             where T : IAsyncApiElement
         {

src/ByteBard.AsyncAPI.Readers/V3/AsyncApiV3VersionService.cs

@@ -182,8 +182,9 @@ public void RegisterComponents(AsyncApiDocument document)
                 this.RegisterComponent(location, item.Value);
             }
 
+            // Register references last, so one pointing at another root channel or server finds it registered.
             string channelBaseUri = "#/channels/";
-            foreach (var channel in document.Channels)
+            foreach (var channel in document.Channels.OrderBy(channel => channel.Value is IAsyncApiReferenceable))
             {
                 var registerableChannelValue = channel.Value;
                 if (channel.Value is IAsyncApiReferenceable reference)
@@ -196,6 +197,12 @@ public void RegisterComponents(AsyncApiDocument document)
                     registerableChannelValue = this.ResolveReference<AsyncApiChannel>(reference.Reference);
                 }
 
+                // A reference to a channel that is not registered (yet) is resolved by the reader later.
+                if (registerableChannelValue == null)
+                {
+                    continue;
+                }
+
                 location = channelBaseUri + JsonPointer.EscapeToken(channel.Key);
                 this.RegisterComponent(location, registerableChannelValue);
 
@@ -206,7 +213,7 @@ public void RegisterComponents(AsyncApiDocument document)
             }
 
             string serverBaseUri = "#/servers/";
-            foreach (var server in document.Servers)
+            foreach (var server in document.Servers.OrderBy(server => server.Value is IAsyncApiReferenceable))
             {
                 var registerableServerValue = server.Value;
                 if (server.Value is IAsyncApiReferenceable serverReference)
@@ -287,6 +294,30 @@ private Uri ToLocationUrl(string location)
         public void SetRootDocument(AsyncApiDocument doc)
         {
             this.RootDocument = doc;
+            doc.SourceWorkspace = this;
+        }
+
+        /// <summary>
+        /// Prepares this workspace for writing the document: registers its components, then adds what the workspace
+        /// the document was read into resolved outside them, so references to those can still be written inline.
+        /// </summary>
+        internal void RegisterForWriting(AsyncApiDocument doc)
+        {
+            this.RootDocument = doc;
+            this.RegisterComponents(doc);
+
+            if (doc.SourceWorkspace == null || doc.SourceWorkspace == this)
+            {
+                return;
+            }
+
+            foreach (var resolved in doc.SourceWorkspace.resolvedReferenceRegistry)
+            {
+                if (!this.resolvedReferenceRegistry.ContainsKey(resolved.Key))
+                {
+                    this.resolvedReferenceRegistry[resolved.Key] = resolved.Value;
+                }
+            }
         }
     }
 }