Add support for dfdl:lengthKind="endOfParent"

daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala

@@ -159,7 +159,7 @@ abstract class ChoiceTermBase(
    * Open issues:
    * 1) Is alignment or leading/trailing skip to be considered syntax. Alignment might not be there.
    * 2) What about an empty sequence that only carries statement annotations such as dfdl:assert or
-   * dfdl:setVariable
+   *    dfdl:setVariable
    *
    * This latter need to be allowed, because while they do not have known required syntax they do
    * have to be executed for side-effect.

daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala

@@ -61,10 +61,8 @@ trait LocalElementMixin extends ParticleMixin with LocalElementGrammarMixin {
           else if (representation =:= Representation.Binary) true
           else false
         }
-        case LengthKind.EndOfParent if isComplexType =>
-          notYetImplemented("lengthKind='endOfParent' for complex type")
-        case LengthKind.EndOfParent =>
-          notYetImplemented("lengthKind='endOfParent' for simple type")
+        // we can rarely statically know if an endOfParent element must have non-zero length
+        case LengthKind.EndOfParent => false
       }
       res
     }.value

daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala

@@ -269,7 +269,6 @@ abstract class SequenceGroupTermBase(xml: Node, lexicalParent: SchemaComponent,
     case SequenceKind.Ordered => true
     case SequenceKind.Unordered => false
   }
-
 }
 
 /**

daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala

@@ -27,9 +27,7 @@ import org.apache.daffodil.lib.iapi.WarnID
 import org.apache.daffodil.lib.schema.annotation.props.Found
 import org.apache.daffodil.lib.schema.annotation.props.NotFound
 import org.apache.daffodil.lib.schema.annotation.props.SeparatorSuppressionPolicy
-import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind
-import org.apache.daffodil.lib.schema.annotation.props.gen.OccursCountKind
-import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo
+import org.apache.daffodil.lib.schema.annotation.props.gen.*
 
 /**
  * Mixin for objects that are shared, but have consistency checks to be run
@@ -562,4 +560,11 @@ trait Term
     }
   }
 
+  final lazy val realElementChildren: Seq[ElementBase] = {
+    termChildren.flatMap {
+      case eb: ElementBase => Seq(eb)
+      case c: Choice => Nil
+      case mg: ModelGroup => mg.realElementChildren
+    }
+  }
 }

daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala

@@ -456,7 +456,15 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
           }
           case LengthKind.Delimited => encodingLengthApprox
           case LengthKind.Pattern => encodingLengthApprox
-          case LengthKind.EndOfParent => LengthMultipleOf(1) // NYI
+          case LengthKind.EndOfParent =>
+            // Technically, the approximate length of an EndOfParent element is the length
+            // of its parent minus the length of prior siblings. However, this value is only
+            // used to compute the ending alignment available to subsequent represented terms.
+            // Per spec (12.3.6), no represented element or model group may follow an
+            // EndOfParent element (only unrepresented elements such as IVC elements are
+            // permitted), so the ending alignment is never consumed. LengthMultipleOf(1)
+            // is a safe conservative value to return.
+            LengthMultipleOf(1)
           // If an element is lengthKind="prefixed", the element's length is the length
           // of the value of the prefix element, which can't be known till runtime
           case LengthKind.Prefixed => LengthMultipleOf(1) // NYI (see DAFFODIL-3066)

daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala

@@ -19,11 +19,14 @@ package org.apache.daffodil.core.grammar
 
 import org.apache.daffodil.core.dsom.Root
 import org.apache.daffodil.core.grammar.primitives.UnicodeByteOrderMark
+import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits
 
 trait RootGrammarMixin
   extends LocalElementGrammarMixin // can be repeating if not root
   { self: Root =>
 
+  requiredEvaluationsAlways(checkEndOfParentRestrictions(Some(LengthUnits.Characters)))
+
   final lazy val document = prod("document") {
     schemaDefinitionUnless(isScalar, "The document element cannot be an array.")
     UnicodeByteOrderMark(this) ~ documentElement

daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala

@@ -17,9 +17,20 @@
 
 package org.apache.daffodil.core.grammar
 
+import org.apache.daffodil.core.dsom.ChoiceTermBase
+import org.apache.daffodil.core.dsom.ElementBase
+import org.apache.daffodil.core.dsom.ModelGroup
+import org.apache.daffodil.core.dsom.Root
+import org.apache.daffodil.core.dsom.SequenceTermBase
 import org.apache.daffodil.core.dsom.Term
 import org.apache.daffodil.core.grammar.primitives.MandatoryTextAlignment
 import org.apache.daffodil.core.runtime1.TermRuntime1Mixin
+import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceLengthKind
+import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind
+import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits
+import org.apache.daffodil.lib.schema.annotation.props.gen.SeparatorPosition
+import org.apache.daffodil.lib.schema.annotation.props.gen.SequenceKind
+import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo
 
 /////////////////////////////////////////////////////////////////
 // Common to all Terms (Elements and ModelGroups)
@@ -84,4 +95,149 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1
     MandatoryTextAlignment(this, knownEncodingAlignmentInBits, true)
   }
 
+  def optEffectiveLengthUnits(optLastNonEOPELU: Option[LengthUnits]): Option[LengthUnits] = {
+    this match {
+      case e: ElementBase =>
+        e.lengthKind match {
+          case LengthKind.EndOfParent => optLastNonEOPELU
+          case LengthKind.Explicit | LengthKind.Prefixed => Some(e.lengthUnits)
+          case LengthKind.Pattern => Some(LengthUnits.Characters)
+          case LengthKind.Implicit | LengthKind.Delimited =>
+            None // invalid parent; SDE fires separately
+        }
+      case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit =>
+        Some(LengthUnits.Bytes)
+      // Sequences are transparent — the ELU is inherited from the nearest enclosing box.
+      case _: SequenceTermBase => optLastNonEOPELU
+      case _: ChoiceTermBase => None // implicit-length choice; SDE fires separately
+    }
+  }
+
+  final lazy val childrenEndOfParent: Seq[ElementBase] = LV(Symbol("childrenEndOfParent")) {
+    val gms = termChildren
+    val chls = gms.flatMap {
+      case eb: ElementBase if eb.lengthKind == LengthKind.EndOfParent => Seq(eb)
+      case eb: ElementBase => Nil
+      case c: ChoiceTermBase => Nil
+      case mg: ModelGroup => mg.childrenEndOfParent
+    }
+    chls
+  }.value
+
+  def checkEndOfParentRestrictions(lastNonEOPELU: Option[LengthUnits]): Boolean = {
+    val term = this
+    lazy val eopChildren = this.childrenEndOfParent
+    lazy val optParentELU = term.optEffectiveLengthUnits(lastNonEOPELU)
+    // checks
+    term match {
+      case rootElem: Root if rootElem.lengthKind == LengthKind.EndOfParent => {
+        rootElem.checkEndOfParentRestrictionsOnCurrentElement(Some(LengthUnits.Characters))
+        eopChildren.foreach { child =>
+          child.checkEndOfParentRestrictionsOnCurrentElement(lastNonEOPELU)
+        }
+      }
+      case e: ElementBase if eopChildren.nonEmpty => {
+        eopChildren.foreach { child =>
+          child.schemaDefinitionWhen(
+            e.lengthKind == LengthKind.Implicit || e.lengthKind == LengthKind.Delimited,
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'."
+          )
+          child.checkEndOfParentRestrictionsOnCurrentElement(optParentELU)
+        }
+      }
+      case s: SequenceTermBase if eopChildren.nonEmpty => {
+        eopChildren.foreach { child =>
+          child.schemaDefinitionWhen(
+            s.separatorPosition == SeparatorPosition.Postfix,
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'."
+          )
+          child.schemaDefinitionWhen(
+            s.sequenceKind != SequenceKind.Ordered,
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'."
+          )
+          child.schemaDefinitionWhen(
+            s.hasTerminator,
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator."
+          )
+          child.schemaDefinitionWhen(
+            s.realElementChildren.exists(e => e.floating == YesNo.Yes),
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'."
+          )
+          child.schemaDefinitionWhen(
+            s.trailingSkip != 0,
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip."
+          )
+        }
+
+      }
+      case c: ChoiceTermBase if eopChildren.nonEmpty => {
+        // TODO: The DFDL spec (12.3.6) explicitly mentions that an EndOfParent element
+        // can be terminated by a choice with choiceLengthKind='explicit', with no reference
+        // to implicit length choices. Later on in 12.3.6, it specified what the parent Effective
+        // Length Units would be for an explicit length choice, again with no reference to
+        // implicit length choices. The only way to get an EndOfParent element within an
+        // implicit length choice would be to have it actually be terminated by the choice's
+        // parent, similar to how we treat elements within an element that is also EndOfParent.
+        // The only mention of implicit length choices in the DFDL Spec is to mention SDEs when
+        // an EndOfParent element is enclosed by a choice with choiceLengthKind='implicit'. We
+        // think that may be a typo, so for now we disallow ChoiceLengthKind.Implicit
+        // enclosing an EndOfParent element.
+        // See Daffodil-3080
+        eopChildren.foreach { child =>
+          child.schemaDefinitionWhen(
+            c.choiceLengthKind == ChoiceLengthKind.Implicit,
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is a choice with dfdl:choiceLengthKind 'implicit'."
+          )
+          child.schemaDefinitionWhen(
+            c.hasTerminator,
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator."
+          )
+          child.schemaDefinitionWhen(
+            c.trailingSkip != 0,
+            "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip."
+          )
+          child.checkEndOfParentRestrictionsOnCurrentElement(optParentELU)
+        }
+      }
+      case _ => // do nothing
+    }
+    // end checks
+    val sawEOP = term.termChildren.foldLeft(false) { case (sawEOP, child) =>
+      if (sawEOP) {
+        // Choice branches are alternatives, not sequential data — the after-EOP SDE must
+        // not fire across branches. Only sequences and elements have sequential ordering.
+        term match {
+          case _: ChoiceTermBase => // has alternatives which can all be EOP; skip
+          case _ =>
+            child.schemaDefinitionWhen(
+              child.isInstanceOf[ModelGroup],
+              "element is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component"
+            )
+            child.schemaDefinitionWhen(
+              child.isRepresented,
+              "element is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component"
+            )
+        }
+      }
+      val res = child.checkEndOfParentRestrictions(optParentELU)
+      term match {
+        // Branches of a choice are alternatives, so should never carry sawEOP from one branch to the next.
+        case _: ChoiceTermBase => false
+        case _ =>
+          child match {
+            case e: ElementBase if e.lengthKind == LengthKind.EndOfParent => true
+            // A non-EOP element forms a hard length boundary. EOP elements nested inside it are
+            // scoped to that element's length, so they must not affect sibling scanning here.
+            case _: ElementBase => false
+            // An explicit-length choice owns a fixed byte span, so EOP elements inside it are
+            // scoped to that span. After the choice ends the parent continues normally.
+            case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => false
+            // Sequences are transparent (no own length boundary),
+            // so EOP state propagates through them.
+            case _ => res
+          }
+      }
+    }
+    sawEOP
+  }
 }

daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala

@@ -20,9 +20,11 @@ package org.apache.daffodil.core.grammar.primitives
 import org.apache.daffodil.core.dsom.ElementBase
 import org.apache.daffodil.core.grammar.Terminal
 import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalBitLimitLengthParser
+import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalEndOfParentLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalKnownLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalRuntimeLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerBitLimitLengthParser
+import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerEndOfParentLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerKnownLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerRuntimeLengthParser
 import org.apache.daffodil.runtime1.processors.unparsers.Unparser
@@ -52,7 +54,18 @@ class BCDIntegerKnownLength(val e: ElementBase, lengthInBits: Long) extends Term
 
 class BCDIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) {
 
-  override lazy val parser = new BCDIntegerBitLimitLengthParser(e.elementRuntimeData)
+  override lazy val parser =
+    new BCDIntegerBitLimitLengthParser(e.elementRuntimeData)
+
+  override lazy val unparser: Unparser = new BCDIntegerMinimumLengthUnparser(
+    e.elementRuntimeData
+  )
+}
+
+class BCDIntegerEndOfParentLength(val e: ElementBase) extends Terminal(e, true) {
+
+  override lazy val parser =
+    new BCDIntegerEndOfParentLengthParser(e.elementRuntimeData)
 
   override lazy val unparser: Unparser = new BCDIntegerMinimumLengthUnparser(
     e.elementRuntimeData
@@ -93,10 +106,19 @@ class BCDDecimalKnownLength(val e: ElementBase, lengthInBits: Long) extends Term
 class BCDDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, true) {
 
   override lazy val parser =
-    new BCDDecimalBitLimitLengthParser(
+    new BCDDecimalBitLimitLengthParser(e.elementRuntimeData, e.binaryDecimalVirtualPoint)
+
+  override lazy val unparser: Unparser =
+    new BCDDecimalMinimumLengthUnparser(
       e.elementRuntimeData,
       e.binaryDecimalVirtualPoint
     )
+}
+
+class BCDDecimalEndOfParentLength(val e: ElementBase) extends Terminal(e, true) {
+
+  override lazy val parser =
+    new BCDDecimalEndOfParentLengthParser(e.elementRuntimeData, e.binaryDecimalVirtualPoint)
 
   override lazy val unparser: Unparser =
     new BCDDecimalMinimumLengthUnparser(

daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala

@@ -20,9 +20,11 @@ package org.apache.daffodil.core.grammar.primitives
 import org.apache.daffodil.core.dsom.ElementBase
 import org.apache.daffodil.core.grammar.Terminal
 import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalBitLimitLengthParser
+import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalEndOfParentLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalKnownLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalRuntimeLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerBitLimitLengthParser
+import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerEndOfParentLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerKnownLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerRuntimeLengthParser
 import org.apache.daffodil.runtime1.processors.unparsers.Unparser
@@ -66,6 +68,15 @@ class IBM4690PackedIntegerPrefixedLength(val e: ElementBase) extends Terminal(e,
   )
 }
 
+class IBM4690PackedIntegerEndOfParentLength(val e: ElementBase) extends Terminal(e, true) {
+  override lazy val parser =
+    new IBM4690PackedIntegerEndOfParentLengthParser(e.elementRuntimeData)
+
+  override lazy val unparser: Unparser = new IBM4690PackedIntegerMinimumLengthUnparser(
+    e.elementRuntimeData
+  )
+}
+
 class IBM4690PackedDecimalRuntimeLength(val e: ElementBase) extends Terminal(e, true) {
   override lazy val parser = new IBM4690PackedDecimalRuntimeLengthParser(
     e.elementRuntimeData,
@@ -115,3 +126,17 @@ class IBM4690PackedDecimalPrefixedLength(val e: ElementBase) extends Terminal(e,
     e.decimalSigned
   )
 }
+
+class IBM4690PackedDecimalEndOfParentLength(val e: ElementBase) extends Terminal(e, true) {
+  override lazy val parser = new IBM4690PackedDecimalEndOfParentLengthParser(
+    e.elementRuntimeData,
+    e.binaryDecimalVirtualPoint,
+    e.decimalSigned
+  )
+
+  override lazy val unparser: Unparser = new IBM4690PackedDecimalMinimumLengthUnparser(
+    e.elementRuntimeData,
+    e.binaryDecimalVirtualPoint,
+    e.decimalSigned
+  )
+}

daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala

@@ -34,6 +34,7 @@ import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerDelimitedParser
 import org.apache.daffodil.runtime1.processors.parsers.BlobSpecifiedLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.HexBinaryDelimitedParser
 import org.apache.daffodil.runtime1.processors.parsers.HexBinaryEndOfBitLimitParser
+import org.apache.daffodil.runtime1.processors.parsers.HexBinaryEndOfParentParser
 import org.apache.daffodil.runtime1.processors.parsers.HexBinarySpecifiedLengthParser
 import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalDelimitedParser
 import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerDelimitedParser
@@ -191,6 +192,16 @@ case class HexBinaryLengthPrefixed(e: ElementBase) extends Terminal(e, true) {
     new HexBinaryMinLengthInBytesUnparser(e.minLength.longValue, e.elementRuntimeData)
 }
 
+case class HexBinaryLengthEndOfParent(e: ElementBase) extends Terminal(e, true) {
+
+  override lazy val parser: DaffodilParser = new HexBinaryEndOfParentParser(
+    e.elementRuntimeData
+  )
+
+  override lazy val unparser: DaffodilUnparser =
+    new HexBinaryMinLengthInBytesUnparser(e.minLength.longValue, e.elementRuntimeData)
+}
+
 abstract class PackedIntegerDelimited(
   e: ElementBase,
   packedSignCodes: PackedSignCodes