From daa666c8adba176021e51bbf5ed29988e85c3315 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Fri, 18 Sep 2026 15:14:43 +0300 Subject: [PATCH 1/4] =?UTF-8?q?fix(sefaria):=20=D7=99=D7=A8=D7=99=D7=93?= =?UTF-8?q?=D7=95=D7=AA=20=D7=A9=D7=95=D7=A8=D7=94=20(
)=20=D7=91=D7=AA?= =?UTF-8?q?=D7=95=D7=9A=20=D7=9E=D7=A7=D7=98=D7=A2=20=D7=A0=D7=A9=D7=9E?= =?UTF-8?q?=D7=A8=D7=95=D7=AA;=20=D7=9B=D7=99=D7=95=D7=95=D7=A5=20=D7=A8?= =?UTF-8?q?=D7=A7=20=D7=91=D7=AA=D7=99=D7=A7=D7=95=D7=A0=D7=99=20=D7=94?= =?UTF-8?q?=D7=96=D7=95=D7=94=D7=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cleanSefariaLine החליפה ברווח כל
שיש אחריו טקסט, בכל ספר מספריא. הכלל נוסף (493939c, #392) בשביל תיקוני הזוהר, שנשברים באמצע משפט לשורות קצרות, אבל ברוב הספרים
הוא מבנה אמיתי: הפרדה בין דיבורים ופסקאות, וכותרת מודגשת בשורה משלה. דווח על אבן האזל על הרמב"ם. סריקה של export ספריא מ-2026-09-18: 315,539 ירידות ב-2,108 ספרים נמחקו. 91.5% מהן באות אחרי סוף משפט, אחרי כותרת מודגשת או כ-

. ב-879 מתוך 944 ספרים (עם 20 ירידות לפחות) פחות מ-10% מהירידות באמצע משפט. עכשיו ה-
הפנימי נשמר, והכיווץ נעשה רק בספרים שברשימה COLLAPSE_INLINE_BREAK_BOOKS (לפי כותרת אנגלית). הרשימה מפורשת ולא נגזרת מסטטיסטיקה של הספר: ספר שקרוב לסף היה עובר צד כשספריא עורכת אותו, וכל שורותיו היו משתנות בעדכון. מפתח השורה מחושב כאילו ה-
הפנימי הוא רווח (rawSegmentForKey). כך מזהי השורות והקישורים לא משתנים, ורק תוכן השורה מתעדכן. בדיקה מוודאת שהצורה המכווצת של שורה שנשמרה זהה לפלט של הניקוי המכווץ. --- .../sefariasqlite/SefariaBookPayloadReader.kt | 5 +- .../sefariasqlite/SefariaImportModels.kt | 6 ++- .../sefariasqlite/SefariaImportText.kt | 39 +++++++++------- .../LineKeyPrefixStabilityTest.kt | 35 ++++++++++++-- .../SefariaCleanSefariaLineTest.kt | 46 ++++++++++++++++--- .../sefariasqlite/SefariaImageEmbedderTest.kt | 2 +- 6 files changed, 104 insertions(+), 29 deletions(-) diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt index fdac5f6b..7e636440 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt @@ -684,7 +684,10 @@ internal class SefariaBookPayloadReader( if (depth == 0 || (leafPrimitive != null && leafPrimitive.isString)) { val content = leafPrimitive?.takeIf { it.isString }?.content if (!content.isNullOrEmpty()) { - val cleaned = SefariaDashlessDibburim.separate(bookHeTitle, cleanSefariaLine(content)) + val cleaned = SefariaDashlessDibburim.separate( + bookHeTitle, + cleanSefariaLine(content, collapseInlineBreaks = collapsesInlineBreaks(bookEnTitle)), + ) if (cleaned.isNotEmpty()) { output += linePrefix + cleaned if (cleanShifts != null) { diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt index 66164d6f..868a182e 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt @@ -234,11 +234,13 @@ internal fun BookPayload.precomputeLineData(): BookPayload { * The line's text with the generated prefix (`(א) `, daf labels…) stripped off. * Inserting one verse reprefixes every later line of the chapter, and hashing * the prefixed text would renumber all of their ids (issue #1211). + * Inline `
` is keyed as a space for the same reason: it is layout, not identity. */ private fun BookPayload.rawSegmentForKey(lineIndex: Int, content: String): String { - val encodedShift = cleanShiftByLineIndex[lineIndex] ?: return content + val encodedShift = cleanShiftByLineIndex[lineIndex] ?: return collapseInlineLineBreaks(content) val prefixLength = generatedPrefixLength(encodedShift) - return if (prefixLength in 1..content.length) content.substring(prefixLength) else content + val raw = if (prefixLength in 1..content.length) content.substring(prefixLength) else content + return collapseInlineLineBreaks(raw) } internal data class VersionMeta( diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt index 82ce3b95..f0df030a 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt @@ -12,14 +12,18 @@ internal fun sanitizeFolder(name: String?): String { // where NN is a two-digit style code. We keep the inner text, drop the marker. private val OTZAR_MARKUP_REGEX = Regex("""@\d{2}([^}]*)\}""") -// Sefaria's merged.json for some books (most notably Tikkunei Zohar from daf -// יז onward) uses `
` tags to mark internal line breaks inside what is -// logically a single paragraph — typically piyut/poetry sections. The app -// renders each `
`-delimited fragment as its own short line, which -// regresses the legacy plain-text Otzaria experience where the paragraph was -// continuous. Collapse them to a single space so paragraphs read as prose. private val HTML_LINE_BREAK_REGEX = Regex("""""", RegexOption.IGNORE_CASE) +// In almost every Sefaria book an inline `
` is real structure (paragraphs, +// a bold heading's own line). These books break mid-sentence instead, into short +// lines that read as broken prose, so their inline breaks collapse to spaces. +private val COLLAPSE_INLINE_BREAK_BOOKS = setOf( + "Tikkunei Zohar", +) + +internal fun collapsesInlineBreaks(bookEnTitle: String): Boolean = + bookEnTitle in COLLAPSE_INLINE_BREAK_BOOKS + // A `
` at the *end* of a line is not an in-paragraph break: Sefaria's MAM // Tanach emits it after `{פ}` to mark an open parasha (פרשה פתוחה), which the // reader renders as the break before the next verse. Collapsing it swallowed @@ -27,26 +31,29 @@ private val HTML_LINE_BREAK_REGEX = Regex("""""", RegexOption.IGNORE_CA private val TRAILING_HTML_LINE_BREAK_REGEX = Regex("""(?:\s*)+\s*$""", RegexOption.IGNORE_CASE) -internal fun cleanSefariaLine(raw: String): String { +internal fun cleanSefariaLine(raw: String, collapseInlineBreaks: Boolean = false): String { var s = if (raw.contains('\n')) raw.replace("\n", "") else raw if (OTZAR_MARKUP_REGEX.containsMatchIn(s)) { s = OTZAR_MARKUP_REGEX.replace(s, "$1") } - if (HTML_LINE_BREAK_REGEX.containsMatchIn(s)) { - val trailing = TRAILING_HTML_LINE_BREAK_REGEX.find(s) - val body = if (trailing != null) s.substring(0, trailing.range.first) else s - // Collapse any double spaces we just introduced - s = HTML_LINE_BREAK_REGEX.replace(body, " ").replace(Regex(" {2,}"), " ").trim() - if (trailing != null && s.isNotEmpty()) { - s += "
" - } - } + s = normalizeLineBreaks(s, inlineBreak = if (collapseInlineBreaks) " " else "
") // Inline any Sefaria textimages as base64 data URIs (no-op if the embedder // hasn't been prefetched or the line contains no such URL). s = SefariaImageEmbedder.substituteImages(s) return s } +/** [line] with every inline `
` as a space — the form a line's id is keyed on. */ +internal fun collapseInlineLineBreaks(line: String): String = normalizeLineBreaks(line, inlineBreak = " ") + +private fun normalizeLineBreaks(line: String, inlineBreak: String): String { + if (!HTML_LINE_BREAK_REGEX.containsMatchIn(line)) return line + val trailing = TRAILING_HTML_LINE_BREAK_REGEX.find(line) + val body = if (trailing != null) line.substring(0, trailing.range.first) else line + val s = HTML_LINE_BREAK_REGEX.replace(body, inlineBreak).replace(Regex(" {2,}"), " ").trim() + return if (trailing != null && s.isNotEmpty()) "$s
" else s +} + // Hebrew label Sefaria's aliyah section name maps to. Named so the alt-TOC // builder can recognise an aliyah level and re-label it by ordinal. internal const val ALIYAH_SECTION_LABEL = "עליה" diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt index f26060f2..1f01ff6c 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt @@ -125,15 +125,44 @@ class LineKeyPrefixStabilityTest { return requireNotNull(payload.precomputed).lineKeyHashes!![lineIndex] } - // The internal
forces cleanSefariaLine to modify this segment. + // The Otzar markup forces cleanSefariaLine to modify this segment. // Inserting a segment before it changes its generated prefix (א -> ב), // but must not change the content key. - val before = cleanedLineHash(listOf("טקסט
שנוקה", "שורה שנייה")) - val after = cleanedLineHash(listOf("שורה חדשה", "טקסט
שנוקה", "שורה שנייה")) + val before = cleanedLineHash(listOf("@04טקסט} שנוקה", "שורה שנייה")) + val after = cleanedLineHash(listOf("שורה חדשה", "@04טקסט} שנוקה", "שורה שנייה")) assertContentEquals(IdAllocatorBindings.lineNaturalKeyHash("טקסט שנוקה"), before) assertContentEquals(before, after, "cleaning must not make the generated prefix part of the key") } + @Test + fun `an inline br is kept in the text but keyed as a space`() { + val json = Json { ignoreUnknownKeys = true } + val reader = SefariaBookPayloadReader(json, Logger.withTag("LineKeyPrefixStabilityTest")) + val schema = json.parseToJsonElement( + """{"depth":1,"sectionNames":["Paragraph"],"addressTypes":["String"]}""", + ).jsonObject + val built = reader.walkTextWithSchema( + schemaObj = schema, + textElement = JsonArray(listOf("כותרת
גוף הדיבור", "שורה שנייה").map(::JsonPrimitive)), + bookHeTitle = "ספר בדיקה", + bookEnTitle = "Test Book", + ) + val lineIndex = built.lines.indexOfFirst { "גוף הדיבור" in it } + assertTrue(built.lines[lineIndex].endsWith("כותרת
גוף הדיבור")) + val payload = BookPayload( + heTitle = "ספר בדיקה", enTitle = "Test Book", categoriesHe = listOf("תנך"), + lines = built.lines, refEntries = built.refs, headings = built.headings, + authors = emptyList(), description = null, heShortDesc = null, + pubDates = emptyList(), altStructures = emptyList(), + cleanShiftByLineIndex = built.cleanShifts, + ).precomputeLineData() + // Books built while inline breaks were collapsed keep their line ids. + assertContentEquals( + IdAllocatorBindings.lineNaturalKeyHash("כותרת גוף הדיבור"), + requireNotNull(payload.precomputed).lineKeyHashes!![lineIndex], + ) + } + @Test fun `a legacy prefixed line without an heRef still migrates`() { // Build 1: pre-#1211 keys — no heRef, so the rendered (prefixed) content. diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt index 0c427c66..3df32ca7 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt @@ -2,6 +2,8 @@ package io.github.kdroidfilter.seforimlibrary.sefariasqlite import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue class SefariaCleanSefariaLineTest { @Test @@ -44,19 +46,50 @@ class SefariaCleanSefariaLineTest { } @Test - fun collapsesBrTagsIntoSpaces() { + fun keepsInlineBrByDefault() { + // Even Ha'azel: each
separates a dibbur from the next one. + assertEquals( + "כותרת ההלכה.
דיבור פירוש.
המשך", + cleanSefariaLine("כותרת ההלכה.
דיבור פירוש.
המשך") + ) + } + + @Test + fun collapsesBrTagsIntoSpacesWhenAsked() { // Sample from Tikkunei Zohar daf יז (idx=33 in merged.json): the - // paragraph is internally split by
tags, which the app used to - // render as short broken lines. + // paragraph is broken mid-sentence into short lines. assertEquals( "ובר מינך לית יחודא בעלאי ותתאי. ואנת אשתמודע אדון על כלא.", - cleanSefariaLine("ובר מינך לית יחודא
בעלאי ותתאי.
ואנת אשתמודע אדון על כלא.") + cleanSefariaLine( + "ובר מינך לית יחודא
בעלאי ותתאי.
ואנת אשתמודע אדון על כלא.", + collapseInlineBreaks = true, + ) ) } @Test fun handlesSelfClosingAndUppercaseBr() { - assertEquals("a b c", cleanSefariaLine("a
b
c")) + assertEquals("a b c", cleanSefariaLine("a
b
c", collapseInlineBreaks = true)) + assertEquals("a
b
c", cleanSefariaLine("a
b
c")) + } + + @Test + fun collapsesOnlyListedBooks() { + assertTrue(collapsesInlineBreaks("Tikkunei Zohar")) + assertFalse(collapsesInlineBreaks("Even Ha'azel on Mishneh Torah, Sabbath")) + } + + @Test + fun keptBreaksCollapseToTheFormerlyCleanedLine() { + // The line key hashes the collapsed form, so it must match what the + // collapsing cleaner produced — otherwise every such line gets a new id. + for (raw in listOf("a
b", "
a

b
", "a
b
", "x
y")) { + assertEquals( + cleanSefariaLine(raw, collapseInlineBreaks = true), + collapseInlineLineBreaks(cleanSefariaLine(raw)), + raw, + ) + } } @Test @@ -71,7 +104,8 @@ class SefariaCleanSefariaLineTest { @Test fun collapsesInnerBrButKeepsTrailingOne() { - assertEquals("a b
", cleanSefariaLine("a
b
")) + assertEquals("a b
", cleanSefariaLine("a
b
", collapseInlineBreaks = true)) + assertEquals("a
b
", cleanSefariaLine("a
b
")) } @Test diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImageEmbedderTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImageEmbedderTest.kt index f7e6075b..6fdc8419 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImageEmbedderTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImageEmbedderTest.kt @@ -464,7 +464,7 @@ class SefariaImageEmbedderTest { ) // Otzar markup,
, and image embedding all in one line val input = """@04פתיחה}
text""" - val out = cleanSefariaLine(input) + val out = cleanSefariaLine(input, collapseInlineBreaks = true) assertTrue(out.startsWith("פתיחה")) assertFalse(out.contains("
")) assertTrue(out.contains("data:image/png;base64,X")) From faf9f87eaf6d2f10621fdd9902a633aa07ea2e4e Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Fri, 18 Sep 2026 15:17:52 +0300 Subject: [PATCH 2/4] =?UTF-8?q?fix(sefaria):=20=D7=9B=D7=99=D7=95=D7=95?= =?UTF-8?q?=D7=A5=20
=20=D7=92=D7=9D=20=D7=91=D7=9E=D7=97=D7=96=D7=95?= =?UTF-8?q?=D7=A8=D7=99=20=D7=A7=D7=95=D7=A8=D7=9F=20=D7=95=D7=91=D7=A1?= =?UTF-8?q?=D7=99=D7=93=D7=95=D7=A8=20=D7=A7=D7=95=D7=A8=D7=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit בשלושת הספרים (מחזור ר"ה, מחזור יוה"כ וסידור קורן השלם, נוסח אשכנז) 65–72% מהירידות באות באמצע משפט, בשורות של כ-50 תווים לפי עימוד הדפוס. ספרים אלה נשארים ברצף אחד, כמו היום. --- .../seforimlibrary/sefariasqlite/SefariaImportText.kt | 4 ++++ .../sefariasqlite/SefariaCleanSefariaLineTest.kt | 1 + 2 files changed, 5 insertions(+) diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt index f0df030a..86e9854b 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt @@ -19,6 +19,10 @@ private val HTML_LINE_BREAK_REGEX = Regex("""""", RegexOption.IGNORE_CA // lines that read as broken prose, so their inline breaks collapse to spaces. private val COLLAPSE_INLINE_BREAK_BOOKS = setOf( "Tikkunei Zohar", + // Koren's print line layout (~50 characters a line). + "The Koren Rosh HaShana Mahzor; Ashkenaz", + "The Koren Yom Kippur Mahzor; Ashkenaz", + "The Koren Shalem Siddur; Ashkenaz", ) internal fun collapsesInlineBreaks(bookEnTitle: String): Boolean = diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt index 3df32ca7..ca768f39 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt @@ -76,6 +76,7 @@ class SefariaCleanSefariaLineTest { @Test fun collapsesOnlyListedBooks() { assertTrue(collapsesInlineBreaks("Tikkunei Zohar")) + assertTrue(collapsesInlineBreaks("The Koren Yom Kippur Mahzor; Ashkenaz")) assertFalse(collapsesInlineBreaks("Even Ha'azel on Mishneh Torah, Sabbath")) } From 9e32cd5b38c7836b1f4334509daeb0ccc08b88ac Mon Sep 17 00:00:00 2001 From: ypl <7353755@gmail.com> Date: Tue, 22 Sep 2026 22:04:27 +0300 Subject: [PATCH 3/4] fix(sefaria): preserve line ids for inline breaks --- .../sefariasqlite/SefariaBookPayloadReader.kt | 42 +++++-- .../sefariasqlite/SefariaDashlessDibburim.kt | 9 +- .../sefariasqlite/SefariaImportModels.kt | 13 ++- .../sefariasqlite/SefariaImportText.kt | 7 +- .../LineKeyPrefixStabilityTest.kt | 107 ++++++++++++++++++ 5 files changed, 157 insertions(+), 21 deletions(-) diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt index 7e636440..99f99eb6 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt @@ -1,6 +1,7 @@ package io.github.kdroidfilter.seforimlibrary.sefariasqlite import co.touchlab.kermit.Logger +import io.github.kdroidfilter.seforimlibrary.common.ids.IdAllocatorBindings import com.fasterxml.jackson.core.JsonFactory import com.fasterxml.jackson.core.JsonToken import io.github.kdroidfilter.seforimlibrary.core.models.PubDate @@ -215,7 +216,7 @@ internal class SefariaBookPayloadReader( listOf(he) + authorTitles.allNameForms(slug) }.distinct() - val (lines, refs, headings, cleanShifts) = buildBookContent( + val (lines, refs, headings, cleanShifts, lineKeyHashOverrides) = buildBookContent( schemaObj = schemaObj, textElement = textElement, bookHeTitle = hebrewTitle, @@ -278,6 +279,7 @@ internal class SefariaBookPayloadReader( titleAliasKeys = titleAliasKeys, singleVersionTitle = singleVersionTitle, cleanShiftByLineIndex = cleanShifts, + lineKeyHashOverrides = lineKeyHashOverrides, versionsMeta = versionsMeta, sourceDirPath = textPath.parent?.toString(), schemaFilePath = schemaPath.toString(), @@ -482,6 +484,7 @@ internal class SefariaBookPayloadReader( val refs: List, val headings: List, val cleanShifts: Map, + val lineKeyHashOverrides: Map, ) /** @@ -515,6 +518,7 @@ internal class SefariaBookPayloadReader( val headings = ArrayList(100) // See BookPayload.cleanShiftByLineIndex — sparse raw-offset bookkeeping. val cleanShifts = HashMap() + val lineKeyHashOverrides = HashMap() fun headingTagForLevel(level: Int): Pair = when (level) { 0 -> "

" to "

" @@ -592,7 +596,8 @@ internal class SefariaBookPayloadReader( referenceableSections = referenceableSections, refIndexOffset = indexOffsets?.top ?: 0, childRefOffsets = indexOffsets?.children, - cleanShifts = cleanShifts + cleanShifts = cleanShifts, + lineKeyHashOverrides = lineKeyHashOverrides, ) } } @@ -642,11 +647,12 @@ internal class SefariaBookPayloadReader( referenceableSections = referenceableSections, refIndexOffset = indexOffsets?.top ?: 0, childRefOffsets = indexOffsets?.children, - cleanShifts = cleanShifts + cleanShifts = cleanShifts, + lineKeyHashOverrides = lineKeyHashOverrides, ) } - return BuiltBookContent(output, refs, headings, cleanShifts) + return BuiltBookContent(output, refs, headings, cleanShifts, lineKeyHashOverrides) } private fun recursiveSections( @@ -673,7 +679,8 @@ internal class SefariaBookPayloadReader( // level (one entry per outer-dim index). childRefOffsets: List? = null, // Sparse raw-offset bookkeeping (see BookPayload.cleanShiftByLineIndex). - cleanShifts: MutableMap? = null + cleanShifts: MutableMap? = null, + lineKeyHashOverrides: MutableMap? = null, ) { // Leaf when depth reached zero, OR when the data is shallower than the // schema declares (e.g. Keter Malkhut: schema says depth=2 but most @@ -684,11 +691,25 @@ internal class SefariaBookPayloadReader( if (depth == 0 || (leafPrimitive != null && leafPrimitive.isString)) { val content = leafPrimitive?.takeIf { it.isString }?.content if (!content.isNullOrEmpty()) { - val cleaned = SefariaDashlessDibburim.separate( - bookHeTitle, - cleanSefariaLine(content, collapseInlineBreaks = collapsesInlineBreaks(bookEnTitle)), - ) + val collapseBreaks = collapsesInlineBreaks(bookEnTitle) + val normalized = cleanSefariaLine(content, collapseInlineBreaks = collapseBreaks) + val cleaned = SefariaDashlessDibburim.separate(bookHeTitle, normalized) if (cleaned.isNotEmpty()) { + // Reproduce the old pipeline BEFORE the dashless repair: keeping + // a break can either enable or suppress that repair. Collapsing + // the final rendered line cannot undo an already inserted dash. + // Store only changed hashes, so plain lines need no extra scan + // during precomputation and we retain no second copy of the text. + if (lineKeyHashOverrides != null && !collapseBreaks && "
" in normalized) { + val keyContent = SefariaDashlessDibburim.separate( + bookHeTitle, + cleanSefariaLine(content, collapseInlineBreaks = true), + recordStats = false, + ) + if (keyContent != cleaned) { + lineKeyHashOverrides[output.size] = IdAllocatorBindings.lineNaturalKeyHash(keyContent) + } + } output += linePrefix + cleaned if (cleanShifts != null) { if (cleaned != content) { @@ -815,7 +836,8 @@ internal class SefariaBookPayloadReader( addressTypes = addressTypes, referenceableSections = referenceableSections, refIndexOffset = nextRefIndexOffset, - cleanShifts = cleanShifts + cleanShifts = cleanShifts, + lineKeyHashOverrides = lineKeyHashOverrides, ) } } diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt index bfbba469..5f49b59c 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt @@ -39,8 +39,11 @@ internal object SefariaDashlessDibburim { private val separatedByBook = ConcurrentHashMap() - /** Returns [line] with its dibbur separated by a dash, or unchanged when the book or line does not fit. */ - fun separate(bookHeTitle: String, line: String): String { + /** + * Returns [line] with its dibbur separated by a dash, or unchanged when it does not fit. + * Set [recordStats] to false for key-only replay of the historic cleaning pipeline. + */ + fun separate(bookHeTitle: String, line: String, recordStats: Boolean = true): String { if (bookHeTitle !in bookHeTitles) return line if (line.startsWith("= 0 -> unchanged segment behind a generated prefix of length n // n < 0 -> cleaned segment; prefix length is -(n + 1), offsets unusable val cleanShiftByLineIndex: Map = emptyMap(), + // Sparse keys for segments whose preserved breaks change the rendered text. + // Computed from the previous cleaning + dashless-repair pipeline, before the + // generated prefix. Generated headings/authors never pass through cleaning. + val lineKeyHashOverrides: Map = emptyMap(), // All [versionTitle, versionSource] pairs from merged.json's `versions` array // (the versions that CONTRIBUTED to the merge). book_version metadata-only // fallback when no per-version sibling files exist. @@ -210,7 +214,7 @@ internal fun BookPayload.precomputeLineData(): BookPayload { val isHeading = BooleanArray(count) for (idx in 0 until count) { val content = lines[idx] - hashes[idx] = IdAllocatorBindings.lineNaturalKeyHash(rawSegmentForKey(idx, content)) + hashes[idx] = lineKeyHashOverrides[idx] ?: IdAllocatorBindings.lineNaturalKeyHash(rawSegmentForKey(idx, content)) legacyHashes[idx] = LegacyLineKey.hash(content, refsByLineIndex[idx]?.heRef) charCounts[idx] = countVisibleChars(content) isHeading[idx] = content.contains("

") || content.contains("

") || @@ -234,13 +238,12 @@ internal fun BookPayload.precomputeLineData(): BookPayload { * The line's text with the generated prefix (`(א) `, daf labels…) stripped off. * Inserting one verse reprefixes every later line of the chapter, and hashing * the prefixed text would renumber all of their ids (issue #1211). - * Inline `
` is keyed as a space for the same reason: it is layout, not identity. + * Segments with preserved breaks use the reader's historic-key override above. */ private fun BookPayload.rawSegmentForKey(lineIndex: Int, content: String): String { - val encodedShift = cleanShiftByLineIndex[lineIndex] ?: return collapseInlineLineBreaks(content) + val encodedShift = cleanShiftByLineIndex[lineIndex] ?: return content val prefixLength = generatedPrefixLength(encodedShift) - val raw = if (prefixLength in 1..content.length) content.substring(prefixLength) else content - return collapseInlineLineBreaks(raw) + return if (prefixLength in 1..content.length) content.substring(prefixLength) else content } internal data class VersionMeta( diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt index 86e9854b..654f213b 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt @@ -12,6 +12,7 @@ internal fun sanitizeFolder(name: String?): String { // where NN is a two-digit style code. We keep the inner text, drop the marker. private val OTZAR_MARKUP_REGEX = Regex("""@\d{2}([^}]*)\}""") +private val REPEATED_SPACES_REGEX = Regex(" {2,}") private val HTML_LINE_BREAK_REGEX = Regex("""""", RegexOption.IGNORE_CASE) // In almost every Sefaria book an inline `
` is real structure (paragraphs, @@ -47,14 +48,14 @@ internal fun cleanSefariaLine(raw: String, collapseInlineBreaks: Boolean = false return s } -/** [line] with every inline `
` as a space — the form a line's id is keyed on. */ +/** [line] with every inline `
` as a space, retaining a terminal break. */ internal fun collapseInlineLineBreaks(line: String): String = normalizeLineBreaks(line, inlineBreak = " ") private fun normalizeLineBreaks(line: String, inlineBreak: String): String { - if (!HTML_LINE_BREAK_REGEX.containsMatchIn(line)) return line + if ('<' !in line || !HTML_LINE_BREAK_REGEX.containsMatchIn(line)) return line val trailing = TRAILING_HTML_LINE_BREAK_REGEX.find(line) val body = if (trailing != null) line.substring(0, trailing.range.first) else line - val s = HTML_LINE_BREAK_REGEX.replace(body, inlineBreak).replace(Regex(" {2,}"), " ").trim() + val s = HTML_LINE_BREAK_REGEX.replace(body, inlineBreak).replace(REPEATED_SPACES_REGEX, " ").trim() return if (trailing != null && s.isNotEmpty()) "$s
" else s } diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt index 1f01ff6c..9281b8a0 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt @@ -5,7 +5,10 @@ import io.github.kdroidfilter.seforimlibrary.common.ids.IdAllocatorBindings import io.github.kdroidfilter.seforimlibrary.common.ids.InMemoryIdAllocator import io.github.kdroidfilter.seforimlibrary.common.ids.LegacyLineKey import io.github.kdroidfilter.seforimlibrary.common.ids.LineOccurrenceCounter +import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.Json +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.jsonObject @@ -121,6 +124,7 @@ class LineKeyPrefixStabilityTest { authors = emptyList(), description = null, heShortDesc = null, pubDates = emptyList(), altStructures = emptyList(), cleanShiftByLineIndex = built.cleanShifts, + lineKeyHashOverrides = built.lineKeyHashOverrides, ).precomputeLineData() return requireNotNull(payload.precomputed).lineKeyHashes!![lineIndex] } @@ -155,6 +159,7 @@ class LineKeyPrefixStabilityTest { authors = emptyList(), description = null, heShortDesc = null, pubDates = emptyList(), altStructures = emptyList(), cleanShiftByLineIndex = built.cleanShifts, + lineKeyHashOverrides = built.lineKeyHashOverrides, ).precomputeLineData() // Books built while inline breaks were collapsed keep their line ids. assertContentEquals( @@ -163,6 +168,108 @@ class LineKeyPrefixStabilityTest { ) } + @Test + fun `preserved breaks do not change keys when dashless repair is suppressed`() { + assertHistoricDashlessKey("דיבור
נוסף. פירוש הדברים") + } + + @Test + fun `preserved breaks do not change keys when dashless repair is newly enabled`() { + assertHistoricDashlessKey("דיבור. פירוש
– המשך") + } + + private fun assertHistoricDashlessKey(raw: String) { + val json = Json { ignoreUnknownKeys = true } + val reader = SefariaBookPayloadReader(json, Logger.withTag("LineKeyPrefixStabilityTest")) + val heTitle = "תוספות על סוכה" + val built = reader.walkTextWithSchema( + schemaObj = json.parseToJsonElement( + """{"depth":1,"sectionNames":["Paragraph"],"addressTypes":["String"]}""", + ).jsonObject, + textElement = JsonArray(listOf(raw, "שורה שנייה").map(::JsonPrimitive)), + bookHeTitle = heTitle, + bookEnTitle = "Tosafot on Sukkah", + ) + val idx = built.refs.first().lineIndex - 1 + assertTrue("
" in built.lines[idx]) + val payload = BookPayload( + heTitle = heTitle, enTitle = "Tosafot on Sukkah", categoriesHe = emptyList(), + lines = built.lines, refEntries = built.refs, headings = built.headings, + authors = emptyList(), description = null, heShortDesc = null, + pubDates = emptyList(), altStructures = emptyList(), + cleanShiftByLineIndex = built.cleanShifts, + lineKeyHashOverrides = built.lineKeyHashOverrides, + ).precomputeLineData() + // The old importer collapsed breaks BEFORE deciding whether to add a dash. + val oldContent = SefariaDashlessDibburim.separate( + heTitle, cleanSefariaLine(raw, collapseInlineBreaks = true), + ) + assertContentEquals( + IdAllocatorBindings.lineNaturalKeyHash(oldContent), + requireNotNull(payload.precomputed).lineKeyHashes!![idx], + "The existing line id must survive both directions of the dashless decision: $raw", + ) + } + + @Test + fun `reader preserves persisted ids through breaks duplicates and prefix shifts`() = runBlocking { + val tempDir = Files.createTempDirectory("inline-br-ids") + try { + val schemaDir = Files.createDirectories(tempDir.resolve("schemas")) + val jsonDir = Files.createDirectories(tempDir.resolve("json")) + val bookDir = Files.createDirectories(jsonDir.resolve("Tosafot_on_Sukkah")) + val heTitle = "תוספות על סוכה" + val enTitle = "Tosafot on Sukkah" + Files.writeString(schemaDir.resolve("$enTitle.json"), """ + {"title":"$enTitle","heTitle":"$heTitle","schema":{ + "title":"$enTitle","heTitle":"$heTitle","depth":1, + "sectionNames":["Paragraph"],"addressTypes":["String"]}} + """.trimIndent()) + val raw = listOf( + "דיבור
נוסף. פירוש הדברים", + "דיבור. פירוש
– המשך", + "כותרת
גוף הדיבור", + "דיבור
נוסף. פירוש הדברים", // same key, distinct occurrence/id + "פרשה פתוחה
", + "שורה ללא תג", + ) + val oldContents = raw.map { + SefariaDashlessDibburim.separate(heTitle, cleanSefariaLine(it, collapseInlineBreaks = true)) + } + val oldAllocator = InMemoryIdAllocator.load(null) + val bookId = oldAllocator.bookId("Sefaria", heTitle) + val oldOccurrences = LineOccurrenceCounter() + val oldIds = oldContents.map { + val hash = IdAllocatorBindings.lineNaturalKeyHash(it) + oldAllocator.lineId(bookId, hash, oldOccurrences.next(bookId, hash)) + } + val state = tempDir.resolve("build_state.db") + oldAllocator.snapshotTo(state) + + val reader = SefariaBookPayloadReader(Json, Logger.withTag("LineKeyPrefixStabilityTest")) + val schemaLookup = reader.buildSchemaLookup(schemaDir) + for (insertAtHead in listOf(false, true)) { + val segments = if (insertAtHead) listOf("מקטע חדש") + raw else raw + Files.writeString(bookDir.resolve("merged.json"), buildJsonObject { + put("title", enTitle) + put("heTitle", heTitle) + put("text", JsonArray(segments.map(::JsonPrimitive))) + }.toString()) + val payload = reader.readBooksInParallel(jsonDir, schemaDir, schemaLookup).single() + .precomputeLineData() + val allocator = InMemoryIdAllocator.load(state) + val ids = allocate(allocator, bookId, payload) + val segmentIds = payload.refEntries.map { ids[it.lineIndex - 1] } + assertEquals(oldIds, if (insertAtHead) segmentIds.drop(1) else segmentIds) + assertEquals(segmentIds.size, segmentIds.toSet().size) + assertEquals(4, payload.lineKeyHashOverrides.size, "store only changed segment keys") + assertTrue(payload.lines.any { "כותרת
גוף הדיבור" in it }) + } + } finally { + tempDir.toFile().deleteRecursively() + } + } + @Test fun `a legacy prefixed line without an heRef still migrates`() { // Build 1: pre-#1211 keys — no heRef, so the rendered (prefixed) content. From 7fa6fa0fc39603a436186549d566195a6147363e Mon Sep 17 00:00:00 2001 From: ypl <7353755@gmail.com> Date: Tue, 22 Sep 2026 22:08:53 +0300 Subject: [PATCH 4/4] fix(sefaria): preserve break-only whitespace behavior --- .../sefariasqlite/SefariaBookPayloadReader.kt | 9 ++++++--- .../seforimlibrary/sefariasqlite/SefariaImportText.kt | 4 ++++ .../sefariasqlite/SefariaVersionsImporter.kt | 1 + .../sefariasqlite/LineKeyPrefixStabilityTest.kt | 5 ++++- .../sefariasqlite/SefariaCleanSefariaLineTest.kt | 3 +++ 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt index 99f99eb6..29048393 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt @@ -497,12 +497,14 @@ internal class SefariaBookPayloadReader( textElement: JsonElement, bookHeTitle: String, bookEnTitle: String, + collectLineKeyOverrides: Boolean = true, ): BuiltBookContent = buildBookContent( schemaObj = schemaObj, textElement = textElement, bookHeTitle = bookHeTitle, bookEnTitle = bookEnTitle, authors = emptyList(), + collectLineKeyOverrides = collectLineKeyOverrides, ) private fun buildBookContent( @@ -510,7 +512,8 @@ internal class SefariaBookPayloadReader( textElement: JsonElement, bookHeTitle: String, bookEnTitle: String, - authors: List + authors: List, + collectLineKeyOverrides: Boolean = true, ): BuiltBookContent { // Pre-allocate with estimated capacity val output = ArrayList(1000) @@ -518,7 +521,7 @@ internal class SefariaBookPayloadReader( val headings = ArrayList(100) // See BookPayload.cleanShiftByLineIndex — sparse raw-offset bookkeeping. val cleanShifts = HashMap() - val lineKeyHashOverrides = HashMap() + val lineKeyHashOverrides = if (collectLineKeyOverrides) HashMap() else null fun headingTagForLevel(level: Int): Pair = when (level) { 0 -> "

" to "

" @@ -652,7 +655,7 @@ internal class SefariaBookPayloadReader( ) } - return BuiltBookContent(output, refs, headings, cleanShifts, lineKeyHashOverrides) + return BuiltBookContent(output, refs, headings, cleanShifts, lineKeyHashOverrides.orEmpty()) } private fun recursiveSections( diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt index 654f213b..e2eba380 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt @@ -53,6 +53,10 @@ internal fun collapseInlineLineBreaks(line: String): String = normalizeLineBreak private fun normalizeLineBreaks(line: String, inlineBreak: String): String { if ('<' !in line || !HTML_LINE_BREAK_REGEX.containsMatchIn(line)) return line + // Kotlin's trim() treats Unicode separators (for example NBSP) as + // whitespace, while java.util.regex \s does not. Preserve the old cleaner's + // behavior for a break-only line before retaining any structural
. + if (HTML_LINE_BREAK_REGEX.replace(line, "").isBlank()) return "" val trailing = TRAILING_HTML_LINE_BREAK_REGEX.find(line) val body = if (trailing != null) line.substring(0, trailing.range.first) else line val s = HTML_LINE_BREAK_REGEX.replace(body, inlineBreak).replace(REPEATED_SPACES_REGEX, " ").trim() diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionsImporter.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionsImporter.kt index 10dc1f47..b8e9515f 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionsImporter.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionsImporter.kt @@ -133,6 +133,7 @@ internal class SefariaVersionsImporter( textElement = textElement, bookHeTitle = payload.heTitle, bookEnTitle = payload.enTitle, + collectLineKeyOverrides = false, ) val versionId = allocator.bookVersionId(input.bookId, versionTitle) val rows = ArrayList(walk.refs.size) diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt index 9281b8a0..c8da16dd 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/LineKeyPrefixStabilityTest.kt @@ -233,6 +233,7 @@ class LineKeyPrefixStabilityTest { "פרשה פתוחה
", "שורה ללא תג", ) + val readerSegments = raw + listOf("
\u00a0", "
\u2003", "
\u00a0
") val oldContents = raw.map { SefariaDashlessDibburim.separate(heTitle, cleanSefariaLine(it, collapseInlineBreaks = true)) } @@ -249,7 +250,7 @@ class LineKeyPrefixStabilityTest { val reader = SefariaBookPayloadReader(Json, Logger.withTag("LineKeyPrefixStabilityTest")) val schemaLookup = reader.buildSchemaLookup(schemaDir) for (insertAtHead in listOf(false, true)) { - val segments = if (insertAtHead) listOf("מקטע חדש") + raw else raw + val segments = if (insertAtHead) listOf("מקטע חדש") + readerSegments else readerSegments Files.writeString(bookDir.resolve("merged.json"), buildJsonObject { put("title", enTitle) put("heTitle", heTitle) @@ -262,6 +263,8 @@ class LineKeyPrefixStabilityTest { val segmentIds = payload.refEntries.map { ids[it.lineIndex - 1] } assertEquals(oldIds, if (insertAtHead) segmentIds.drop(1) else segmentIds) assertEquals(segmentIds.size, segmentIds.toSet().size) + assertEquals(raw.size + if (insertAtHead) 1 else 0, payload.refEntries.size, + "break-only lines with Unicode whitespace must not create refs") assertEquals(4, payload.lineKeyHashOverrides.size, "store only changed segment keys") assertTrue(payload.lines.any { "כותרת
גוף הדיבור" in it }) } diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt index ca768f39..dc1c5df3 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt @@ -112,5 +112,8 @@ class SefariaCleanSefariaLineTest { @Test fun dropsBrOnlyLine() { assertEquals("", cleanSefariaLine("

")) + assertEquals("", cleanSefariaLine("
\u00a0")) + assertEquals("", cleanSefariaLine("
\u2003")) + assertEquals("", cleanSefariaLine("
\u00a0
")) } }