Include annotation for packets from FPP in dictionary

compiler/lib/src/main/scala/analysis/Analysis.scala

@@ -88,14 +88,14 @@ case class Analysis(
   tlmPacketSet: Option[TlmPacketSet] = None,
   /** The mapping from nodes to implied uses */
   impliedUseMap: Map[AstNode.Id, ImpliedUse.Uses] = Map(),
-  /** Node ID of the template expansion currently */
-  template: Option[AstNode.Id] = None,
-  /** Maps template expansion AST nodes to their */
-  templateExpansionMap: Map[AstNode.Id, TemplateExpansion] = Map(),
-  /** Tracks the current expansion we are analyzing inside of */
+  /** Node ID of the current template definition */
+  templateDefId: Option[AstNode.Id] = None,
+  /** The current template expansion */
   templateExpansion: Option[TemplateExpansion] = None,
-  /** Node ID of the template definition we are currently cloning inside of */
-  templateDefinition: Option[AstNode.Id] = None,
+  /** Node ID of the current template expansion */
+  templateExpansionId: Option[AstNode.Id] = None,
+  /** Mapping from template expansion nodes to their expansions */
+  templateExpansionMap: Map[AstNode.Id, TemplateExpansion] = Map(),
   /** The set of symbols defined with a dictionary specifier */
   dictionarySymbolSet: Set[Symbol] = Set(),
   /** The mapping from system symbols to systems */

compiler/lib/src/main/scala/analysis/Analyzers/ModuleAnalyzer.scala

@@ -17,20 +17,24 @@ trait ModuleAnalyzer extends Analyzer {
     yield a2.copy(scopeNameList = a.scopeNameList)
   }
 
-  override def specTemplateExpandAnnotatedNode(
+  override def specTemplateExpandAnnotatedNodeExpanded(
     a: Analysis,
-    aNode: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]]
+    aNode: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]],
+    members: List[Ast.ModuleMember]
   ) = {
     val (_, node, _) = aNode
-    // Check if this template has been expanded
-    node.data.members match {
-      case Some(members) => {
-        // Mark the analysis with the proper expansion
-        val a1 = a.copy(templateExpansion = Some(a.templateExpansionMap(node.id)))
+    a.templateExpansionMap.get(node.id) match {
+      case Some(expansion) =>
+        // This node has been expanded by analysis.
+        // Mark the analysis with the proper expansion.
+        val a1 = a.copy(templateExpansion = Some(expansion))
         for { a2 <- visitList(a1, members, matchModuleMember) }
         yield a2.copy(templateExpansion = a.templateExpansion)
-      }
-      case None => Right(a)
+      case None =>
+        // This node has not yet been expanded by analysis.
+        // The expansion is provided in the source model, in the form
+        // expand T { ... }.
+        Right(a)
     }
   }
 

compiler/lib/src/main/scala/analysis/CheckSemantics/CheckTemplateDefs.scala

@@ -0,0 +1,53 @@
+package fpp.compiler.analysis
+
+import fpp.compiler.ast._
+import fpp.compiler.util._
+
+/** Check template definitions */
+object CheckTemplateDefs
+  extends Analyzer
+  with ModuleAnalyzer
+{
+
+  override def defModuleTemplateAnnotatedNode(
+    a: Analysis,
+    aNode: Ast.Annotated[AstNode[Ast.DefModuleTemplate]]
+  ) = {
+    val (_, node, _) = aNode
+    val data = node.data
+    for {
+      _ <- a.templateDefId match {
+        case Some(defId) => Left(
+          SemanticError.TemplateDefError(
+            Locations.get(node.id),
+            "module template definitions cannot be nested",
+            Locations.get(defId)
+          )
+        )
+        case None => Right(())
+      }
+      a <- Right(a.copy(templateDefId = Some(node.id)))
+      a <- visitList(a, data.members, matchModuleMember)
+      a <- Right(a.copy(templateDefId = None))
+    } yield a
+  }
+
+  override def specTemplateExpandAnnotatedNode(
+    a: Analysis,
+    aNode: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]]
+  ) = {
+    val (_, node, _) = aNode
+    val data = node.data
+    a.templateDefId match {
+      case Some(defId) => Left(
+        SemanticError.TemplateDefError(
+          Locations.get(node.id),
+          "template expansion specifiers cannot appear inside a module template",
+          Locations.get(defId)
+        )
+      )
+      case None => Right(a)
+    }
+  }
+
+}

compiler/lib/src/main/scala/analysis/CheckSemantics/CheckTemplateExpansions.scala

@@ -0,0 +1,54 @@
+package fpp.compiler.analysis
+
+import fpp.compiler.ast._
+import fpp.compiler.util._
+
+/** Check template expansions */
+object CheckTemplateExpansions
+  extends Analyzer
+  with ModuleAnalyzer
+{
+
+  override def defModuleTemplateAnnotatedNode(
+    a: Analysis,
+    aNode: Ast.Annotated[AstNode[Ast.DefModuleTemplate]]
+  ) =
+    a.templateExpansionId match {
+      case Some(expandId) => Left(
+        SemanticError.TemplateExpandError(
+          Locations.get(aNode._2.id),
+          "module template definitions cannot appear inside a template expansion",
+          Locations.get(expandId)
+        )
+      )
+      case None => Right(a)
+    }
+
+  override def specTemplateExpandAnnotatedNodeExpanded(
+    a: Analysis,
+    aNode: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]],
+    members: List[Ast.ModuleMember]
+  ) =
+    for {
+      a <- specTemplateExpandAnnotatedNodeUnexpanded(a, aNode)
+      a <- Right(a.copy(templateExpansionId = Some(aNode._2.id)))
+      a <- visitList(a, members, matchModuleMember)
+      a <- Right(a.copy(templateExpansionId = None))
+    } yield a
+
+  override def specTemplateExpandAnnotatedNodeUnexpanded(
+    a: Analysis,
+    aNode: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]],
+  ) =
+    a.templateExpansionId match {
+      case Some(defId) => Left(
+        SemanticError.TemplateExpandError(
+          Locations.get(aNode._2.id),
+          "template expansions cannot be nested",
+          Locations.get(defId)
+        )
+      )
+      case None => Right(a)
+    }
+
+}

compiler/lib/src/main/scala/analysis/CheckSemantics/ExpandSyntaxAndEnterSymbols.scala

@@ -16,8 +16,10 @@ object ExpandSyntaxAndEnterSymbols {
       (a, tul) <- ResolveSpecInclude.transUnitList(a, tul)
       tul <- AddStateEnums.transUnitList(tul)
       a <- EnterSymbols.visitList(a, tul, EnterSymbols.transUnit)
+      a <- CheckTemplateDefs.visitList(a, tul, CheckTemplateDefs.transUnit)
+      a <- CheckTemplateExpansions.visitList(a, tul, CheckTemplateExpansions.transUnit)
       a <- CheckTemplateUses.visitList(a, tul, CheckTemplateUses.transUnit)
-      (_, tul) <- ExpandTemplates.transformList(a, tul, ExpandTemplates.transUnit)
+      tul <- ExpandTemplates.transformList(a, tul, ExpandTemplates.transUnit)
       a <- EnterTemplateExpansionSymbols.visitList(a, tul, EnterTemplateExpansionSymbols.transUnit)
     } yield (a, tul)
   }

compiler/lib/src/main/scala/analysis/Semantics/TemplateExpansion.scala

@@ -3,7 +3,7 @@ package fpp.compiler.analysis
 import fpp.compiler.ast._
 import fpp.compiler.util._
 
-/** An FPP template */
+/** An FPP template expansion */
 case class TemplateExpansion(
   /** The AST node defining the template */
   defNode: Ast.Annotated[AstNode[Ast.DefModuleTemplate]],

compiler/lib/src/main/scala/ast/AstStateTransformer.scala

@@ -35,6 +35,15 @@ trait AstStateTransformer extends AstTransformer {
     yield (pair._1, pair._2.reverse)
   }
 
+  /** Transform a list and flatten the result */
+  def transformFlatList[A, B](
+    s: State,
+    list: List[A],
+    transform: (In, A) => Result[List[B]]
+  ): Result[List[B]] =
+    for ((s, bs) <- transformList(s, list, transform))
+      yield (s, bs.flatten)
+
   def tuMember(s: State, member: Ast.TUMember): Result[List[Ast.TUMember]] =
     matchModuleMember(s, member)
 

compiler/lib/src/main/scala/ast/AstTransformer.scala

@@ -288,6 +288,22 @@ trait AstTransformer {
   def specTemplateExpandAnnotatedNode(
     in: In,
     node: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]]
+  ): ResultAnnotatedNode[Ast.SpecTemplateExpand] =
+    node._2.data.members match {
+      case Some(members) => specTemplateExpandAnnotatedNodeExpanded(in, node, members)
+      case None => specTemplateExpandAnnotatedNodeUnexpanded(in, node)
+    }
+
+  def specTemplateExpandAnnotatedNodeUnexpanded(
+    in: In,
+    node: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]]
+  ): ResultAnnotatedNode[Ast.SpecTemplateExpand] =
+    defaultAnnotatedNode(in, node)
+
+  def specTemplateExpandAnnotatedNodeExpanded(
+    in: In,
+    node: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]],
+    members: List[Ast.ModuleMember]
   ): ResultAnnotatedNode[Ast.SpecTemplateExpand] =
     defaultAnnotatedNode(in, node)
 

compiler/lib/src/main/scala/ast/AstUnitTransformer.scala

@@ -0,0 +1,42 @@
+package fpp.compiler.transform
+
+import fpp.compiler.analysis._
+import fpp.compiler.ast._
+import fpp.compiler.util._
+
+/** Transform an AST, producing a unit Out value */
+trait AstUnitTransformer extends AstTransformer
+{
+
+  type Out = Unit
+
+  /** Transform a list in sequence */
+  def transformList[A, B](
+    in: In,
+    list: List[A],
+    transform: (In, A) => Result[B]
+  ): Result.Result[List[B]] = {
+    def helper(inList: List[A], outList: List[B]): Result.Result[List[B]] = {
+      inList match {
+        case Nil => Right(outList)
+        case head :: tail => transform(in, head) match {
+          case Left(e) => Left(e)
+          case Right((_, list)) => helper(tail, list :: outList)
+        }
+      }
+    }
+    helper(list, Nil).map(_.reverse)
+  }
+
+  /** Transform a list and flatten the result */
+  def transformFlatList[A, B](
+    in: In,
+    list: List[A],
+    transform: (In, A) => Result[List[B]]
+  ): Result.Result[List[B]] = transformList(in, list, transform).map(_.flatten)
+
+  /** Adapt a simple result function to the form of the visitor functions */
+  def resultAdapter[A,B](f: (A, B) => Result.Result[B]) =
+    (a: A, b: B) => for (c <- f(a, b)) yield ((), c)
+
+}

compiler/lib/src/main/scala/ast/ModuleStateTransformer.scala

@@ -11,15 +11,29 @@ trait ModuleStateTransformer extends AstStateTransformer {
   ) = {
     val (pre, node, post) = aNode
     val Ast.DefModule(name, members) = node.data
-    for { result <- transformList(s, members, moduleMember) }
+    for ((s1, members1) <- transformFlatList(s, members, moduleMember))
     yield {
-      val (s1, members1) = result
-      val defModule = Ast.DefModule(name, members1.flatten)
+      val defModule = Ast.DefModule(name, members1)
       val node1 = AstNode.create(defModule, node.id)
       (s1, (pre, node1, post))
     }
   }
 
+  override def specTemplateExpandAnnotatedNodeExpanded(
+    s: State,
+    aNode: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]],
+    members: List[Ast.ModuleMember]
+  ) = {
+    val (pre, node, post) = aNode
+    val Ast.SpecTemplateExpand(template, args, _) = node.data
+    for ((s1, members1) <- transformFlatList(s, members, moduleMember))
+      yield {
+        val specTemplateExpand = Ast.SpecTemplateExpand(template, args, Some(members1))
+        val node1 = AstNode.create(specTemplateExpand, node.id)
+        (s1, (pre, node1, post))
+      }
+  }
+
   def moduleMember(s: State, member: Ast.ModuleMember): Result[List[Ast.ModuleMember]] =
     matchModuleMember(s, member)
 

compiler/lib/src/main/scala/codegen/AstWriter.scala

@@ -186,31 +186,17 @@ object AstWriter extends AstVisitor with LineUtils {
     ).map(indentIn)
   }
 
-  override def specTemplateExpandAnnotatedNodeUnexpanded(
+  override def specTemplateExpandAnnotatedNode(
     in: In,
     aNode: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]]
-  ) = {
-    val (_, node, _) = aNode
-    val data = node.data
-    lines("expand") ++
-    List.concat(
-      qualIdent(data.template.data),
-      templateArgList(data.args)
-    ).map(indentIn)
-  }
-
-  override def specTemplateExpandAnnotatedNodeExpanded(
-    in: In,
-    aNode: Ast.Annotated[AstNode[Ast.SpecTemplateExpand]],
-    members: List[Ast.ModuleMember]
   ) = {
     val (_, node, _) = aNode
     val data = node.data
     lines("expand") ++
     List.concat(
       qualIdent(data.template.data),
       templateArgList(data.args),
-      members.flatMap(moduleMember)
+      data.members.getOrElse(Nil).flatMap(moduleMember)
     ).map(indentIn)
   }
 

compiler/lib/src/main/scala/codegen/DictionaryJsonWriter/DictionaryJsonEncoder.scala

@@ -568,10 +568,12 @@ case class DictionaryJsonEncoder(
     /** JSON Encoding for Telemetry Packet Sets */
     private implicit def tlmPacketSetEncoder: Encoder[(Name.Unqualified, TlmPacketSet)] = new Encoder[(Name.Unqualified, TlmPacketSet)] {
         override def apply(entry: (Name.Unqualified, TlmPacketSet)): Json = {
-            Json.obj(
+            val (setPreA, _, setPostA) = entry._2.aNode
+            val json = Json.obj(
                 "name" -> entry._1.toString.asJson,
                 "members" -> entry._2.packetMap.map((id, packet) => {
-                    Json.obj(
+                    val (preA, _, postA) = packet.aNode
+                    val packetJson = Json.obj(
                         "name" -> packet.getName.asJson,
                         "id" -> id.asJson,
                         "group" -> packet.group.asJson,
@@ -580,12 +582,14 @@ case class DictionaryJsonEncoder(
                             s"${e.instance.toString}.${e.tlmChannel.getName}"
                         }).asJson
                     )
+                    jsonWithOptionalValues(packetJson, Map("annotation" -> concatAnnotations(preA, postA)))
                 }).asJson,
                 "omitted" -> entry._2.omittedIdSet.map(tlmId => {
                     val e = dictionary.tlmChannelEntryMap(tlmId)
                     s"${e.instance.toString}.${e.tlmChannel.getName}"
                 }).toList.sorted.asJson
             )
+            jsonWithOptionalValues(json, Map("annotation" -> concatAnnotations(setPreA, setPostA)))
         }
     }
 

compiler/lib/src/main/scala/syntax/Parser.scala

@@ -1096,8 +1096,10 @@ object Parser extends Parsers {
   val templateArgList = argList(templateArg)
 
   def specTemplateExpand: Parser[Ast.SpecTemplateExpand] = {
-    (expand ~>! node(qualIdent)) ~! templateArgList ^^ {
-      case id ~ args => Ast.SpecTemplateExpand(id, args, None)
+    (expand ~>! node(qualIdent)) ~! templateArgList ~! opt(
+      lbrace ~>! moduleMembers <~! rbrace
+    ) ^^ {
+      case id ~ args ~ membersOpt => Ast.SpecTemplateExpand(id, args, membersOpt)
     }
   }