fix(sefaria): ירידות שורה (<br>) בתוך מקטע נשמרות; כיווץ רק בתיקוני הזוהר ובקורן

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt

@@ -1,6 +1,7 @@
 package io.github.kdroidfilter.seforimlibrary.sefariasqlite
 
 import co.touchlab.kermit.Logger
+import io.github.kdroidfilter.seforimlibrary.common.ids.IdAllocatorBindings
 import com.fasterxml.jackson.core.JsonFactory
 import com.fasterxml.jackson.core.JsonToken
 import io.github.kdroidfilter.seforimlibrary.core.models.PubDate
@@ -215,7 +216,7 @@ internal class SefariaBookPayloadReader(
                 listOf(he) + authorTitles.allNameForms(slug)
             }.distinct()
 
-            val (lines, refs, headings, cleanShifts) = buildBookContent(
+            val (lines, refs, headings, cleanShifts, lineKeyHashOverrides) = buildBookContent(
                 schemaObj = schemaObj,
                 textElement = textElement,
                 bookHeTitle = hebrewTitle,
@@ -278,6 +279,7 @@ internal class SefariaBookPayloadReader(
                 titleAliasKeys = titleAliasKeys,
                 singleVersionTitle = singleVersionTitle,
                 cleanShiftByLineIndex = cleanShifts,
+                lineKeyHashOverrides = lineKeyHashOverrides,
                 versionsMeta = versionsMeta,
                 sourceDirPath = textPath.parent?.toString(),
                 schemaFilePath = schemaPath.toString(),
@@ -482,6 +484,7 @@ internal class SefariaBookPayloadReader(
         val refs: List<RefEntry>,
         val headings: List<Heading>,
         val cleanShifts: Map<Int, Int>,
+        val lineKeyHashOverrides: Map<Int, ByteArray>,
     )
 
     /**
@@ -494,27 +497,31 @@ internal class SefariaBookPayloadReader(
         textElement: JsonElement,
         bookHeTitle: String,
         bookEnTitle: String,
+        collectLineKeyOverrides: Boolean = true,
     ): BuiltBookContent = buildBookContent(
         schemaObj = schemaObj,
         textElement = textElement,
         bookHeTitle = bookHeTitle,
         bookEnTitle = bookEnTitle,
         authors = emptyList(),
+        collectLineKeyOverrides = collectLineKeyOverrides,
     )
 
     private fun buildBookContent(
         schemaObj: JsonObject,
         textElement: JsonElement,
         bookHeTitle: String,
         bookEnTitle: String,
-        authors: List<String>
+        authors: List<String>,
+        collectLineKeyOverrides: Boolean = true,
     ): BuiltBookContent {
         // Pre-allocate with estimated capacity
         val output = ArrayList<String>(1000)
         val refs = ArrayList<RefEntry>(1000)
         val headings = ArrayList<Heading>(100)
         // See BookPayload.cleanShiftByLineIndex — sparse raw-offset bookkeeping.
         val cleanShifts = HashMap<Int, Int>()
+        val lineKeyHashOverrides = if (collectLineKeyOverrides) HashMap<Int, ByteArray>() else null
 
         fun headingTagForLevel(level: Int): Pair<String, String> = when (level) {
             0 -> "<h1>" to "</h1>"
@@ -592,7 +599,8 @@ internal class SefariaBookPayloadReader(
                     referenceableSections = referenceableSections,
                     refIndexOffset = indexOffsets?.top ?: 0,
                     childRefOffsets = indexOffsets?.children,
-                    cleanShifts = cleanShifts
+                    cleanShifts = cleanShifts,
+                    lineKeyHashOverrides = lineKeyHashOverrides,
                 )
             }
         }
@@ -642,11 +650,12 @@ internal class SefariaBookPayloadReader(
                 referenceableSections = referenceableSections,
                 refIndexOffset = indexOffsets?.top ?: 0,
                 childRefOffsets = indexOffsets?.children,
-                cleanShifts = cleanShifts
+                cleanShifts = cleanShifts,
+                lineKeyHashOverrides = lineKeyHashOverrides,
             )
         }
 
-        return BuiltBookContent(output, refs, headings, cleanShifts)
+        return BuiltBookContent(output, refs, headings, cleanShifts, lineKeyHashOverrides.orEmpty())
     }
 
     private fun recursiveSections(
@@ -673,7 +682,8 @@ internal class SefariaBookPayloadReader(
         // level (one entry per outer-dim index).
         childRefOffsets: List<Int>? = null,
         // Sparse raw-offset bookkeeping (see BookPayload.cleanShiftByLineIndex).
-        cleanShifts: MutableMap<Int, Int>? = null
+        cleanShifts: MutableMap<Int, Int>? = null,
+        lineKeyHashOverrides: MutableMap<Int, ByteArray>? = null,
     ) {
         // Leaf when depth reached zero, OR when the data is shallower than the
         // schema declares (e.g. Keter Malkhut: schema says depth=2 but most
@@ -684,8 +694,25 @@ internal class SefariaBookPayloadReader(
         if (depth == 0 || (leafPrimitive != null && leafPrimitive.isString)) {
             val content = leafPrimitive?.takeIf { it.isString }?.content
             if (!content.isNullOrEmpty()) {
-                val cleaned = SefariaDashlessDibburim.separate(bookHeTitle, cleanSefariaLine(content))
+                val collapseBreaks = collapsesInlineBreaks(bookEnTitle)
+                val normalized = cleanSefariaLine(content, collapseInlineBreaks = collapseBreaks)
+                val cleaned = SefariaDashlessDibburim.separate(bookHeTitle, normalized)
                 if (cleaned.isNotEmpty()) {
+                    // Reproduce the old pipeline BEFORE the dashless repair: keeping
+                    // a break can either enable or suppress that repair. Collapsing
+                    // the final rendered line cannot undo an already inserted dash.
+                    // Store only changed hashes, so plain lines need no extra scan
+                    // during precomputation and we retain no second copy of the text.
+                    if (lineKeyHashOverrides != null && !collapseBreaks && "<br>" in normalized) {
+                        val keyContent = SefariaDashlessDibburim.separate(
+                            bookHeTitle,
+                            cleanSefariaLine(content, collapseInlineBreaks = true),
+                            recordStats = false,
+                        )
+                        if (keyContent != cleaned) {
+                            lineKeyHashOverrides[output.size] = IdAllocatorBindings.lineNaturalKeyHash(keyContent)
+                        }
+                    }
                     output += linePrefix + cleaned
                     if (cleanShifts != null) {
                         if (cleaned != content) {
@@ -812,7 +839,8 @@ internal class SefariaBookPayloadReader(
                 addressTypes = addressTypes,
                 referenceableSections = referenceableSections,
                 refIndexOffset = nextRefIndexOffset,
-                cleanShifts = cleanShifts
+                cleanShifts = cleanShifts,
+                lineKeyHashOverrides = lineKeyHashOverrides,
             )
         }
     }

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt

@@ -39,8 +39,11 @@ internal object SefariaDashlessDibburim {
 
     private val separatedByBook = ConcurrentHashMap<String, Int>()
 
-    /** Returns [line] with its dibbur separated by a dash, or unchanged when the book or line does not fit. */
-    fun separate(bookHeTitle: String, line: String): String {
+    /**
+     * Returns [line] with its dibbur separated by a dash, or unchanged when it does not fit.
+     * Set [recordStats] to false for key-only replay of the historic cleaning pipeline.
+     */
+    fun separate(bookHeTitle: String, line: String, recordStats: Boolean = true): String {
         if (bookHeTitle !in bookHeTitles) return line
         if (line.startsWith("<h", ignoreCase = true) || SPACED_DASH.containsMatchIn(line)) return line
         val cut = line.indexOf(". ")
@@ -56,7 +59,7 @@ internal object SefariaDashlessDibburim {
         // particular, structural markers such as `מתני'` and `(הג"ה` must not
         // be rewritten merely because they happen to end with a period.
         if (DhExtractor.extract(separated, DhExtractor.Format.DASH) == null) return line
-        separatedByBook.merge(bookHeTitle, 1, Int::plus)
+        if (recordStats) separatedByBook.merge(bookHeTitle, 1, Int::plus)
         return separated
     }
 

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt

@@ -113,6 +113,10 @@ internal data class BookPayload(
     //   n >= 0          -> unchanged segment behind a generated prefix of length n
     //   n < 0           -> cleaned segment; prefix length is -(n + 1), offsets unusable
     val cleanShiftByLineIndex: Map<Int, Int> = emptyMap(),
+    // Sparse keys for segments whose preserved breaks change the rendered text.
+    // Computed from the previous cleaning + dashless-repair pipeline, before the
+    // generated prefix. Generated headings/authors never pass through cleaning.
+    val lineKeyHashOverrides: Map<Int, ByteArray> = emptyMap(),
     // All [versionTitle, versionSource] pairs from merged.json's `versions` array
     // (the versions that CONTRIBUTED to the merge). book_version metadata-only
     // fallback when no per-version sibling files exist.
@@ -210,7 +214,7 @@ internal fun BookPayload.precomputeLineData(): BookPayload {
     val isHeading = BooleanArray(count)
     for (idx in 0 until count) {
         val content = lines[idx]
-        hashes[idx] = IdAllocatorBindings.lineNaturalKeyHash(rawSegmentForKey(idx, content))
+        hashes[idx] = lineKeyHashOverrides[idx] ?: IdAllocatorBindings.lineNaturalKeyHash(rawSegmentForKey(idx, content))
         legacyHashes[idx] = LegacyLineKey.hash(content, refsByLineIndex[idx]?.heRef)
         charCounts[idx] = countVisibleChars(content)
         isHeading[idx] = content.contains("<h1>") || content.contains("<h2>") ||
@@ -234,6 +238,7 @@ internal fun BookPayload.precomputeLineData(): BookPayload {
  * The line's text with the generated prefix (`(א) `, daf labels…) stripped off.
  * Inserting one verse reprefixes every later line of the chapter, and hashing
  * the prefixed text would renumber all of their ids (issue #1211).
+ * Segments with preserved breaks use the reader's historic-key override above.
  */
 private fun BookPayload.rawSegmentForKey(lineIndex: Int, content: String): String {
     val encodedShift = cleanShiftByLineIndex[lineIndex] ?: return content

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt

@@ -12,41 +12,57 @@ internal fun sanitizeFolder(name: String?): String {
 // where NN is a two-digit style code. We keep the inner text, drop the marker.
 private val OTZAR_MARKUP_REGEX = Regex("""@\d{2}([^}]*)\}""")
 
-// Sefaria's merged.json for some books (most notably Tikkunei Zohar from daf
-// יז onward) uses `<br>` tags to mark internal line breaks inside what is
-// logically a single paragraph — typically piyut/poetry sections. The app
-// renders each `<br>`-delimited fragment as its own short line, which
-// regresses the legacy plain-text Otzaria experience where the paragraph was
-// continuous. Collapse them to a single space so paragraphs read as prose.
+private val REPEATED_SPACES_REGEX = Regex(" {2,}")
 private val HTML_LINE_BREAK_REGEX = Regex("""<br\s*/?>""", RegexOption.IGNORE_CASE)
 
+// In almost every Sefaria book an inline `<br>` is real structure (paragraphs,
+// a bold heading's own line). These books break mid-sentence instead, into short
+// lines that read as broken prose, so their inline breaks collapse to spaces.
+private val COLLAPSE_INLINE_BREAK_BOOKS = setOf(
+    "Tikkunei Zohar",
+    // Koren's print line layout (~50 characters a line).
+    "The Koren Rosh HaShana Mahzor; Ashkenaz",
+    "The Koren Yom Kippur Mahzor; Ashkenaz",
+    "The Koren Shalem Siddur; Ashkenaz",
+)
+
+internal fun collapsesInlineBreaks(bookEnTitle: String): Boolean =
+    bookEnTitle in COLLAPSE_INLINE_BREAK_BOOKS
+
 // A `<br>` at the *end* of a line is not an in-paragraph break: Sefaria's MAM
 // Tanach emits it after `{פ}` to mark an open parasha (פרשה פתוחה), which the
 // reader renders as the break before the next verse. Collapsing it swallowed
 // that break, so only breaks with text after them are collapsed.
 private val TRAILING_HTML_LINE_BREAK_REGEX =
     Regex("""(?:\s*<br\s*/?>)+\s*$""", RegexOption.IGNORE_CASE)
 
-internal fun cleanSefariaLine(raw: String): String {
+internal fun cleanSefariaLine(raw: String, collapseInlineBreaks: Boolean = false): String {
     var s = if (raw.contains('\n')) raw.replace("\n", "") else raw
     if (OTZAR_MARKUP_REGEX.containsMatchIn(s)) {
         s = OTZAR_MARKUP_REGEX.replace(s, "$1")
     }
-    if (HTML_LINE_BREAK_REGEX.containsMatchIn(s)) {
-        val trailing = TRAILING_HTML_LINE_BREAK_REGEX.find(s)
-        val body = if (trailing != null) s.substring(0, trailing.range.first) else s
-        // Collapse any double spaces we just introduced
-        s = HTML_LINE_BREAK_REGEX.replace(body, " ").replace(Regex(" {2,}"), " ").trim()
-        if (trailing != null && s.isNotEmpty()) {
-            s += "<br>"
-        }
-    }
+    s = normalizeLineBreaks(s, inlineBreak = if (collapseInlineBreaks) " " else "<br>")
     // Inline any Sefaria textimages as base64 data URIs (no-op if the embedder
     // hasn't been prefetched or the line contains no such URL).
     s = SefariaImageEmbedder.substituteImages(s)
     return s
 }
 
+/** [line] with every inline `<br>` as a space, retaining a terminal break. */
+internal fun collapseInlineLineBreaks(line: String): String = normalizeLineBreaks(line, inlineBreak = " ")
+
+private fun normalizeLineBreaks(line: String, inlineBreak: String): String {
+    if ('<' !in line || !HTML_LINE_BREAK_REGEX.containsMatchIn(line)) return line
+    // Kotlin's trim() treats Unicode separators (for example NBSP) as
+    // whitespace, while java.util.regex \s does not. Preserve the old cleaner's
+    // behavior for a break-only line before retaining any structural <br>.
+    if (HTML_LINE_BREAK_REGEX.replace(line, "").isBlank()) return ""
+    val trailing = TRAILING_HTML_LINE_BREAK_REGEX.find(line)
+    val body = if (trailing != null) line.substring(0, trailing.range.first) else line
+    val s = HTML_LINE_BREAK_REGEX.replace(body, inlineBreak).replace(REPEATED_SPACES_REGEX, " ").trim()
+    return if (trailing != null && s.isNotEmpty()) "$s<br>" else s
+}
+
 // Hebrew label Sefaria's aliyah section name maps to. Named so the alt-TOC
 // builder can recognise an aliyah level and re-label it by ordinal.
 internal const val ALIYAH_SECTION_LABEL = "עליה"

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionsImporter.kt

@@ -133,6 +133,7 @@ internal class SefariaVersionsImporter(
                 textElement = textElement,
                 bookHeTitle = payload.heTitle,
                 bookEnTitle = payload.enTitle,
+                collectLineKeyOverrides = false,
             )
             val versionId = allocator.bookVersionId(input.bookId, versionTitle)
             val rows = ArrayList<VersionLine>(walk.refs.size)