From 7c76359b34537c2a93f0a69e75a1fe263698c7a6 Mon Sep 17 00:00:00 2001 From: Kgothatso Ngako Date: Fri, 2 Oct 2026 19:48:22 +0200 Subject: [PATCH] feat(chapters): a chunk is a markdown block, not a blank-line paragraph Phases 7 and 8 of docs/chapter-uploads.md. Measured against markdown-it -- the parser mantra.press translates with -- this app's blank-line split disagreed on ten cases in eleven, and on the text as well as the boundaries. markdown-it's content is a leaf block's inline source with the BLOCK marker removed, so `# Chapter one` is "Chapter one" while inline markup is kept. That is the half that matters to a translator: handed "# Chapter one" they can break the document by mistyping the hash, and handed "Chapter one" they cannot. So splitBlocks adopts it, boundaries and stripped text both, with three deliberate departures. Code blocks yield no chunk -- markdown-it's fence and code_block carry content, so mantra.press asks a translator to translate `code();` and the group to sign it as prose. A table is one chunk of its own source rather than one per cell, because four envelopes to carry one word each is a poor trade and a table's alignment is structure per-cell translation cannot preserve. And a link reference definition yields nothing, which is free: it is markdown-it's own behaviour with the `reference` rule left enabled, where mantra.press disables it -- reaching for a block rule to stop links rendering -- and makes a translatable chunk of a URL. Hand-rolled, not org.jetbrains:markdown, for one reason specific to this function: its output is hashed into ids a quorum signs, so a version bump that moved one boundary would silently re-split every chapter proposed afterwards and nothing would notice. What pins it instead is thirty golden fixtures generated FROM markdown-it by docs/scripts/markdown-block-fixtures.js, which also encodes the three departures so the Kotlin test is plain equality. MarkdownProducer is new and is what makes stripping markers safe: nothing in the app could put a `#` back before. It substitutes by index, which is strictly better than the thing it mirrors -- mantra.press matches on text, so a chapter with two identical paragraphs puts the first one's translation into both. The scanner segments every line rather than only the chunks, which is what makes the byte-for-byte round trip exact by construction rather than by getting every markup rule right. ProposedEvent.summarize stops re-splitting a chapter to count its chunks and takes the count from the batch. That number is a count of the batch dressed as a fact about the text: across a splitter change the figure in the approval gate would disagree with the events being approved, and it re-scanned every open proposal's prose on every emission of the room's list. Phase 2's constant was flagged as provisional pending this, and measured it did not move: prose separated by blank lines splits the same under both rules, so the fixture pinning 24 KiB over forty paragraphs is the shape this does not touch. The narrower claim holds -- a heading- or list-heavy chapter now reaches both caps sooner, visibly, in the form's own counter. 987 tests, no failures; m3Audit's budgets all met. Co-Authored-By: Claude Opus 5 --- .../compose/nostr/nip30303/ChapterProposal.kt | 2 +- .../compose/nostr/nip30303/ChunkEvent.kt | 8 +- .../press/mantra/compose/text/Markdown.kt | 33 +- .../mantra/compose/text/MarkdownBlocks.kt | 337 ++++++++++++++++++ .../mantra/compose/text/MarkdownProducer.kt | 86 +++++ .../mantra/compose/text/ProposedEvent.kt | 22 +- .../compose/ui/composable/AddChapterScreen.kt | 14 +- .../ui/composable/FrostSigningScreen.kt | 11 +- .../ui/view/model/AddChapterViewModel.kt | 8 +- .../ui/view/model/ProposalListViewModel.kt | 11 +- .../compose/ui/view/state/UploadReview.kt | 2 +- .../nostr/nip30303/ChapterChunkSplitTest.kt | 2 +- .../nostr/nip30303/ChapterProposalSizeTest.kt | 10 +- .../mantra/compose/text/ChapterTextTest.kt | 2 +- .../compose/text/MarkdownBlockFixtures.kt | 258 ++++++++++++++ .../mantra/compose/text/MarkdownBlockTest.kt | 36 ++ .../compose/text/MarkdownProducerTest.kt | 155 ++++++++ .../compose/ui/view/state/UploadReviewTest.kt | 5 +- docs/chapter-uploads.md | 42 ++- docs/scripts/markdown-block-fixtures.js | 177 +++++++++ 20 files changed, 1177 insertions(+), 44 deletions(-) create mode 100644 composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownBlocks.kt create mode 100644 composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownProducer.kt create mode 100644 composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt create mode 100644 composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockTest.kt create mode 100644 composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownProducerTest.kt create mode 100644 docs/scripts/markdown-block-fixtures.js diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposal.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposal.kt index 0fb718c5..3961d80c 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposal.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposal.kt @@ -66,7 +66,7 @@ object ChapterProposal { (chunks.size + 1) * EVENT_OVERHEAD_BYTES fun proposalBytes(originalText: String): Int = - proposalBytes(originalText, Markdown.splitParagraphs(originalText)) + proposalBytes(originalText, Markdown.splitChunks(originalText)) fun fits(originalText: String, chunks: List): Boolean = proposalBytes(originalText, chunks) <= MAX_PROPOSAL_BYTES diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChunkEvent.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChunkEvent.kt index 48876fc9..9c363b2f 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChunkEvent.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChunkEvent.kt @@ -63,13 +63,13 @@ class ChunkEvent( sig = chapter.sig ).originalText() ?: return emptyList() - return Markdown.splitParagraphs(originalText).mapIndexed { index, paragraph -> + return Markdown.splitChunks(originalText).mapIndexed { index, chunk -> build( chapterId = chapter.id, - text = paragraph, + text = chunk, index = index, - wordCount = Markdown.wordCount(paragraph), - characterCount = Markdown.characterCount(paragraph), + wordCount = Markdown.wordCount(chunk), + characterCount = Markdown.characterCount(chunk), createdAt = chapter.createdAt, ) } diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/Markdown.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/Markdown.kt index 5771c2dd..30a5cc86 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/Markdown.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/Markdown.kt @@ -9,8 +9,6 @@ package press.mantra.compose.text object Markdown { private val WHITESPACE = Regex("\\s+") - /** A paragraph break is one or more blank lines. */ - private val PARAGRAPH_BREAK = Regex("(?:\\r?\\n){2,}") fun wordCount(text: String): Int { val trimmed = text.trim() @@ -20,11 +18,30 @@ object Markdown { fun characterCount(text: String): Int = text.length /** - * Split markdown into paragraphs (blocks separated by blank lines), - * trimmed and with empties removed. Each paragraph becomes a chunk. + * The blocks a chapter's markdown splits into, one chunk each. + * + * A chunk is a **leaf** block's inline text with its block marker removed: + * the words a translator is given and the words the group signs. Containers + * -- lists, blockquotes -- contribute nothing of their own; a thematic + * break, a code block and a link reference definition contribute nothing at + * all. + * + * This is markdown-it's block token stream, which is what mantra.press + * translates against, with the three departures recorded in + * `docs/chapter-uploads.md`. It replaced a blank-line split, and the two + * disagreed on ten cases in eleven -- on the text as well as the + * boundaries, because markdown-it strips the marker, so `# Chapter one` is + * `"Chapter one"`. Inline markup is kept verbatim: emphasis and links are + * the translator's to carry, where a `#` mistyped into a translation breaks + * the document. + * + * Its output is hashed into chunk ids a quorum signs, so it is pinned by + * `MarkdownBlockFixtures` -- generated from markdown-it itself -- rather + * than by this comment. See [MarkdownBlocks] for the subset it recognises. */ - fun splitParagraphs(text: String): List = - text.split(PARAGRAPH_BREAK) - .map { it.trim() } - .filter { it.isNotEmpty() } + fun splitBlocks(text: String): List = + MarkdownBlocks.segment(text).mapNotNull { it.block } + + /** Just the words of each chunk, which is all an event carries. */ + fun splitChunks(text: String): List = splitBlocks(text).map { it.text } } diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownBlocks.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownBlocks.kt new file mode 100644 index 00000000..99751009 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownBlocks.kt @@ -0,0 +1,337 @@ +package press.mantra.compose.text + +/** + * What kind of leaf block a chunk came out of. + * + * Carried so [MarkdownProducer] can put the block's marker back when the + * chunk's words have been replaced by a translation. Nothing else reads it. + */ +enum class BlockKind { + Paragraph, + Heading, + SetextHeading, + ListItem, + Quote, + + /** + * A whole table, as its own source. + * + * One of the three deliberate departures from markdown-it, which gives a + * chunk per cell. See `docs/chapter-uploads.md`. + */ + Table, +} + +/** + * One chunk, as the words and the marker that were in front of them. + * + * [text] is what a translator is given and what the group signs: the leaf + * block's inline source with the **block** marker removed -- no `#`, no `> `, + * no `- `, no setext underline -- while inline markup (`*italic*`, a link's + * brackets) is kept verbatim. The first is structure a translation should not + * be able to break by mistyping; the second is the translator's to carry. + * + * [markup] is the marker that was removed, kept so a document can be put back + * together. For an ATX heading it is the run of `#`; for a list item the + * bullet or the ordinal; for a blockquote `>`. It is not stored on the chunk + * event -- it is recoverable from the chapter's own `original_text`, which is + * where [MarkdownProducer] reads it from. + */ +data class MarkdownBlock( + val kind: BlockKind, + val markup: String, + val text: String, + /** The indent the block sat at, so a nested list item goes back where it was. */ + val indent: String = "", +) + +/** + * A run of lines, and the chunk it is -- or nothing, if it is not prose. + * + * A segmentation rather than a split, because two things need it. [block] being + * null is a fenced code block, a thematic break, a link reference definition or + * a blank line: material that carries no chunk and still has to be in the + * document afterwards. + * + * [lines] is the exact source, so a document with no translations applied is + * reassembled byte for byte by joining every segment's lines with `\n`. + */ +data class MarkdownSegment( + val lines: List, + val block: MarkdownBlock?, +) + +/** + * The block structure of a chapter's markdown. + * + * A documented **subset** of CommonMark, and hand-rolled rather than taken from + * a library on purpose: its output is hashed into chunk ids that a quorum + * signs, so a dependency bump that moved one boundary would silently re-split + * every chapter proposed afterwards and nothing in the app would notice. What + * pins it instead is `MarkdownBlockFixtures`, generated from markdown-it by + * `docs/scripts/markdown-block-fixtures.js`. + * + * What it recognises: ATX and setext headings, paragraphs with lazy + * continuation, blockquotes, bullet and ordered list items including one level + * of nesting, fenced and indented code, thematic breaks, tables, and link + * reference definitions. + * + * What it does not, each of which has a fixture saying what happens instead: + * HTML blocks (markdown-it's default is `html: false`, so a `
` is a + * paragraph on both sides), footnotes, and blockquotes containing lists. + */ +internal object MarkdownBlocks { + + private val ATX = Regex("^ {0,3}(#{1,6})(?:[ \t]+(.*?))?[ \t]*#*[ \t]*$") + private val SETEXT_UNDERLINE = Regex("^ {0,3}(=+|-+)[ \t]*$") + private val THEMATIC_BREAK = Regex("^ {0,3}((\\*[ \t]*){3,}|(-[ \t]*){3,}|(_[ \t]*){3,})$") + private val FENCE = Regex("^ {0,3}(`{3,}|~{3,})(.*)$") + private val BULLET = Regex("^( {0,3})([-*+])([ \t]+)(.*)$") + private val ORDERED = Regex("^( {0,3})(\\d{1,9}[.)])([ \t]+)(.*)$") + private val QUOTE = Regex("^ {0,3}>[ \t]?(.*)$") + private val REFERENCE_DEFINITION = Regex("^ {0,3}\\[[^\\]]+\\]:[ \t]*\\S.*$") + private val INDENTED_CODE = Regex("^ {4,}\\S.*$") + private val TABLE_DELIMITER = Regex("^ {0,3}\\|?[ \t]*:?-{1,}:?[ \t]*(\\|[ \t]*:?-{1,}:?[ \t]*)*\\|?[ \t]*$") + + fun segment(text: String): List { + val lines = text.split("\n") + val segments = mutableListOf() + var i = 0 + + while (i < lines.size) { + val line = lines[i] + + when { + line.isBlank() -> { + segments += MarkdownSegment(listOf(line), null) + i++ + } + + FENCE.matches(line) -> { + val fence = FENCE.find(line)!!.groupValues[1] + val taken = mutableListOf(lines[i]) + i++ + while (i < lines.size) { + taken += lines[i] + val closing = FENCE.find(lines[i])?.groupValues?.get(1) + i++ + if (closing != null && closing.length >= fence.length) break + } + // Departure 1: code is not prose, so no chunk. + segments += MarkdownSegment(taken, null) + } + + THEMATIC_BREAK.matches(line) -> { + segments += MarkdownSegment(listOf(line), null) + i++ + } + + REFERENCE_DEFINITION.matches(line) -> { + // Departure 3, which is markdown-it's own behaviour with the + // `reference` rule enabled: a definition is not prose. + segments += MarkdownSegment(listOf(line), null) + i++ + } + + INDENTED_CODE.matches(line) -> { + val taken = mutableListOf() + while (i < lines.size && (INDENTED_CODE.matches(lines[i]) || lines[i].isBlank())) { + // A blank line only belongs to the code block if more code + // follows it; otherwise it is the separator after it. + if (lines[i].isBlank() && + lines.drop(i + 1).firstOrNull { it.isNotBlank() } + ?.let { !INDENTED_CODE.matches(it) } != false + ) { + break + } + taken += lines[i] + i++ + } + segments += MarkdownSegment(taken, null) + } + + ATX.matches(line) -> { + val match = ATX.find(line)!! + segments += MarkdownSegment( + lines = listOf(line), + block = MarkdownBlock( + kind = BlockKind.Heading, + markup = match.groupValues[1], + text = match.groupValues[2].trim(), + ), + ) + i++ + } + + isTableStart(lines, i) -> { + val taken = mutableListOf() + while (i < lines.size && lines[i].isNotBlank() && lines[i].contains('|')) { + taken += lines[i] + i++ + } + // Departure 2: one chunk of the whole table's source. + segments += MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.Table, + markup = "", + text = taken.joinToString("\n"), + ), + ) + } + + QUOTE.matches(line) -> { + val taken = mutableListOf() + val inner = mutableListOf() + while (i < lines.size && QUOTE.matches(lines[i])) { + taken += lines[i] + inner += QUOTE.find(lines[i])!!.groupValues[1] + i++ + } + // A `>` on its own splits one blockquote into two paragraphs, + // which markdown-it reads as two inline runs and so two chunks. + val paragraphs = inner.splitOnBlank() + if (paragraphs.size <= 1) { + segments += MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.Quote, + markup = ">", + text = inner.joinToString("\n").trim(), + ), + ) + } else { + // Re-segmented per inner paragraph, so each is its own + // chunk and the source still reassembles exactly. + var consumed = 0 + paragraphs.forEachIndexed { index, paragraph -> + val span = paragraph.size + if (index < paragraphs.size - 1) 1 else 0 + segments += MarkdownSegment( + lines = taken.subList(consumed, consumed + span).toList(), + block = MarkdownBlock( + kind = BlockKind.Quote, + markup = ">", + text = paragraph.joinToString("\n").trim(), + ), + ) + consumed += span + } + } + } + + BULLET.matches(line) || ORDERED.matches(line) -> { + val match = (BULLET.find(line) ?: ORDERED.find(line))!! + val indent = match.groupValues[1] + val marker = match.groupValues[2] + val gap = match.groupValues[3] + val taken = mutableListOf(line) + val content = mutableListOf(match.groupValues[4]) + i++ + // Lazy continuation: a plain line under an item belongs to it. + // A new marker, at any indent, is a new item -- which is what + // makes a nested list two chunks rather than one. + while (i < lines.size && isLazyContinuation(lines[i])) { + taken += lines[i] + content += lines[i].trim() + i++ + } + segments += MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.ListItem, + markup = marker + gap, + text = content.joinToString("\n").trim(), + indent = indent, + ), + ) + } + + else -> { + // A paragraph, unless the line after it underlines it. + val taken = mutableListOf(line) + i++ + var setext: String? = null + while (i < lines.size) { + val next = lines[i] + if (SETEXT_UNDERLINE.matches(next) && !THEMATIC_BREAK.matches(next)) { + setext = SETEXT_UNDERLINE.find(next)!!.groupValues[1] + taken += next + i++ + break + } + if (!isLazyContinuation(next)) break + taken += next + i++ + } + + segments += if (setext != null) { + MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.SetextHeading, + markup = setext, + text = taken.dropLast(1).joinToString("\n").trim(), + ), + ) + } else { + MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.Paragraph, + markup = "", + text = taken.joinToString("\n").trim(), + ), + ) + } + } + } + } + + return segments + } + + /** + * Whether [line] continues the block above it rather than starting one. + * + * The list is every block start this scanner recognises. A line that starts + * none of them, and is not blank, is prose belonging to whatever came + * before -- which is what markdown-it calls a lazy continuation and is why + * `First line.\nSecond line.` is one chunk and not two. + */ + private fun isLazyContinuation(line: String): Boolean = when { + line.isBlank() -> false + ATX.matches(line) -> false + THEMATIC_BREAK.matches(line) -> false + FENCE.matches(line) -> false + QUOTE.matches(line) -> false + BULLET.matches(line) -> false + ORDERED.matches(line) -> false + REFERENCE_DEFINITION.matches(line) -> false + else -> true + } + + /** A table is a row of cells with a delimiter row under it. */ + private fun isTableStart(lines: List, at: Int): Boolean = + lines[at].contains('|') && + at + 1 < lines.size && + lines[at + 1].contains('-') && + TABLE_DELIMITER.matches(lines[at + 1]) + + /** Splits a blockquote's inner lines into its paragraphs. */ + private fun List.splitOnBlank(): List> { + val out = mutableListOf>() + var current = mutableListOf() + forEach { line -> + if (line.isBlank()) { + if (current.isNotEmpty()) { + out += current + current = mutableListOf() + } + } else { + current += line + } + } + if (current.isNotEmpty()) out += current + return out + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownProducer.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownProducer.kt new file mode 100644 index 00000000..155345b7 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownProducer.kt @@ -0,0 +1,86 @@ +package press.mantra.compose.text + +/** + * A chapter's markdown with translations in place of its words. + * + * The other half of [Markdown.splitBlocks], and the piece that makes stripping + * block markers safe. A chunk carries `"Chapter one"` rather than + * `"# Chapter one"` so that a translator cannot break the document by mistyping + * the `#` -- which is only defensible if something can put the `#` back. Before + * this, nothing in the app could: there was no equivalent of mantra.press's + * `lib/markdown-producer/index.js` at all. + * + * **Substitution is by index, not by text**, and that is strictly better than + * the thing it mirrors. mantra.press matches with + * `translationChunks.find(tc => tc.text == token.content)`, so a chapter with + * two identical paragraphs puts the first one's translation into both of them. + * A chunk here already carries an `IndexTag`, so position is available and + * exact. + */ +object MarkdownProducer { + + /** + * [originalText] rebuilt with [translations] substituted by chunk index. + * + * A chunk with no translation keeps its own source **verbatim** -- not + * re-rendered from its parsed form -- so a chapter with no translations at + * all comes back byte for byte. That is the property worth having: it means + * every markup-reassembly bug is one failing assertion away rather than a + * difference somebody has to notice in a rendered document. + * + * A translated chunk is rendered from its [MarkdownBlock.kind]: the marker + * goes back in front, and for a blockquote in front of every line, so a + * translation that grew a line is still quoted. + */ + fun produce(originalText: String, translations: Map): String { + var chunkIndex = 0 + + return MarkdownBlocks.segment(originalText).flatMap { segment -> + val block = segment.block ?: return@flatMap segment.lines + + val translation = translations[chunkIndex] + chunkIndex++ + + if (translation.isNullOrEmpty()) segment.lines else render(block, translation) + }.joinToString("\n") + } + + /** How many chunks [originalText] has, which is what a caller indexes against. */ + fun chunkCount(originalText: String): Int = Markdown.splitBlocks(originalText).size + + private fun render(block: MarkdownBlock, translation: String): List = + when (block.kind) { + BlockKind.Paragraph -> translation.lines() + + BlockKind.Heading -> listOf("${block.markup} $translation") + + // The underline has to be as long as the text, or it stops being an + // underline -- a one-character `=` under a translated title is a + // setext heading in CommonMark but reads as a typo. + BlockKind.SetextHeading -> { + // The underline matches the line immediately above it, which is + // the last line of the translation rather than its first: a `=` + // shorter than the text it underlines is still a setext heading + // in CommonMark but reads as a typo. + val lines = translation.lines() + lines + block.markup.first().toString() + .repeat(lines.last().length.coerceAtLeast(1)) + } + + BlockKind.ListItem -> { + val lines = translation.lines() + listOf("${block.indent}${block.markup}${lines.first()}") + + // A continuation line is indented under the marker, so the + // item stays one item rather than becoming a paragraph after + // it. + lines.drop(1).map { "${block.indent}${" ".repeat(block.markup.length)}$it" } + } + + BlockKind.Quote -> translation.lines().map { "> $it" } + + // A table's chunk is its own source, so its translation is too: there + // is no marker to put back, and re-deriving the pipes from a + // translated table would be guessing at its columns. + BlockKind.Table -> translation.lines() + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/ProposedEvent.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/ProposedEvent.kt index 7b11f466..a13bcba6 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/ProposedEvent.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/ProposedEvent.kt @@ -31,7 +31,25 @@ object ProposedEvent { */ data class Summary(val label: String, val detail: String) - fun summarize(event: Event): Summary = when (event.kind) { + /** + * [chunkCount] is how many chunks travel with a chapter in the same batch, + * which only a caller holding the batch can know. + * + * It used to be derived here, by re-splitting the chapter's own text. That + * was wrong twice over. It is a *count of the batch* dressed as a fact about + * the text, so across a splitter change the number a member reads in the + * approval gate disagrees with the chunks they are actually approving -- the + * batch was built by the proposer's splitter and counted by the reader's. + * And it re-split the whole markdown of every chapter on every emission of + * the room's proposal list, which at twenty open proposals is a few hundred + * kilobytes of prose re-scanned to produce a number already sitting in the + * items beside it. + * + * Null where the caller has no batch -- a transcript line, a signed event on + * its own -- and the summary then says nothing about chunks rather than + * guessing. + */ + fun summarize(event: Event, chunkCount: Int? = null): Summary = when (event.kind) { DialectEvent.KIND -> Summary( label = "New dialect", detail = DialectEvent( @@ -74,8 +92,6 @@ object ProposedEvent { // is a whole chapter, so its size stands in for it here. The chunk // count says how many of the batch's other items came out of this // text, which is the rest of what is being signed. - val chunkCount = chapter.originalText()?.let { Markdown.splitParagraphs(it).size } - listOfNotNull( chapter.name(), chapter.wordCount()?.let { "$it words" }, diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/AddChapterScreen.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/AddChapterScreen.kt index 98ee337a..4ff53dc8 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/AddChapterScreen.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/AddChapterScreen.kt @@ -168,8 +168,8 @@ fun AddChapterScreen( val originalText = originalTextFieldState.text.toString() val wordCount = Markdown.wordCount(originalText) val characterCount = Markdown.characterCount(originalText) - val chunks = Markdown.splitParagraphs(originalText) - val paragraphCount = chunks.size + val chunks = Markdown.splitChunks(originalText) + val chunkCount = chunks.size // A chapter hangs off a version, and the group has to be able to // sign; without both there is nothing this screen can propose. @@ -179,7 +179,7 @@ fun AddChapterScreen( // and a session signs a bounded number of events. Past that the // chapter has to be split in two, which is worth saying while there // is still a cursor in the text rather than after a failed propose. - val tooManyChunks = paragraphCount > AddChapterViewModel.MAX_CHUNKS_PER_CHAPTER + val tooManyChunks = chunkCount > AddChapterViewModel.MAX_CHUNKS_PER_CHAPTER // The other cap, and the one that had been enforced nowhere: a // proposal carries the chapter whole and every chunk cut from @@ -191,7 +191,7 @@ fun AddChapterScreen( val canProposeChapter = artifactVersion != null && addChapterUIState.canSign && - paragraphCount > 0 && + chunkCount > 0 && !tooManyChunks && !tooLargeToPropose @@ -268,7 +268,7 @@ fun AddChapterScreen( Text( text = stringResource( Res.string.d_chunks_is_more_than_the_group_can_sign, - paragraphCount, + chunkCount, AddChapterViewModel.MAX_CHUNKS_PER_CHAPTER, ), style = MaterialTheme.typography.bodySmall, @@ -278,7 +278,7 @@ fun AddChapterScreen( Text( text = stringResource( Res.string.this_chapter_is_too_long_to_propose, - paragraphCount, + chunkCount, stringResource(Res.string.s_kib, kibibytes(proposalBytes)), stringResource( Res.string.s_kib, @@ -337,7 +337,7 @@ fun AddChapterScreen( kibibytes(ChapterProposal.MAX_PROPOSAL_BYTES), ), ), - paragraphCount, + chunkCount, AddChapterViewModel.MAX_CHUNKS_PER_CHAPTER, ), style = MaterialTheme.typography.labelMedium, diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/FrostSigningScreen.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/FrostSigningScreen.kt index fbecea82..ec1fb4b9 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/FrostSigningScreen.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/FrostSigningScreen.kt @@ -48,6 +48,7 @@ import press.mantra.compose.database.model.types.FrostSigningStage import press.mantra.compose.managers.FrostSigningManager import press.mantra.compose.repository.ChatRepository import press.mantra.compose.repository.FrostSigningRepository +import press.mantra.compose.nostr.nip30303.ChunkEvent import press.mantra.compose.text.ProposedEvent import press.mantra.compose.ui.composable.widgets.profile.ProfileAvatar import press.mantra.compose.ui.theme.MantraTheme @@ -358,10 +359,14 @@ private fun WhatIsBeingSigned(events: List, expected: Int) { ) } + // The chunk count a chapter reports is a count of this batch, so it + // comes from here rather than from re-reading the chapter's text. + val chunkCount = events.count { it.kind == ChunkEvent.KIND } + events.forEachIndexed { index, event -> if (index > 0) HorizontalDivider() - OneThingBeingSigned(event) + OneThingBeingSigned(event, chunkCount) } Text( @@ -376,8 +381,8 @@ private fun WhatIsBeingSigned(events: List, expected: Int) { /** One event of the batch, described as the thing it is. */ @Composable -private fun OneThingBeingSigned(event: Event) { - val summary = ProposedEvent.summarize(event) +private fun OneThingBeingSigned(event: Event, chunkCount: Int) { + val summary = ProposedEvent.summarize(event, chunkCount) Column(verticalArrangement = Arrangement.spacedBy(MaterialTheme.spacing.space50)) { Text(text = summary.label, style = MaterialTheme.typography.labelMedium) diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/AddChapterViewModel.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/AddChapterViewModel.kt index bea91925..8bde3330 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/AddChapterViewModel.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/AddChapterViewModel.kt @@ -102,7 +102,7 @@ class AddChapterViewModel( ) { val name = nameField.text.toString() val originalText = originalTextField.text.toString() - val paragraphs = Markdown.splitParagraphs(originalText) + val chunks = Markdown.splitChunks(originalText) // Both caps are the group's, not this screen's, and proposing past // either throws rather than failing softly. The form says so as it is @@ -113,9 +113,9 @@ class AddChapterViewModel( // weighs. A chapter of sixty-three one-word paragraphs passes the second // and fails the first; one of three very long ones does the reverse. if (name.isBlank() || - paragraphs.isEmpty() || - paragraphs.size > MAX_CHUNKS_PER_CHAPTER || - !ChapterProposal.fits(originalText, paragraphs) + chunks.isEmpty() || + chunks.size > MAX_CHUNKS_PER_CHAPTER || + !ChapterProposal.fits(originalText, chunks) ) { onFailure.invoke() return diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/ProposalListViewModel.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/ProposalListViewModel.kt index 938d3d83..e6e40587 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/ProposalListViewModel.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/ProposalListViewModel.kt @@ -18,6 +18,7 @@ import press.mantra.compose.extensions.shortened import press.mantra.compose.managers.FrostSigningManager import press.mantra.compose.repository.ChatRepository import press.mantra.compose.repository.FrostSigningRepository +import press.mantra.compose.nostr.nip30303.ChunkEvent import press.mantra.compose.text.ProposedEvent import press.mantra.compose.ui.view.state.ProposalListUIState @@ -86,7 +87,15 @@ class ProposalListViewModel( else -> local.proposer?.humanReadableNameOrPubkey() ?: local.session.coordinatorPublicKey.shortened() }, - signs = events.map { ProposedEvent.summarize(it) }, + // Counted from the batch rather than derived from the chapter's + // text -- the number a member reads has to be the number of events + // they are approving. See ProposedEvent.summarize. + signs = events.map { event -> + ProposedEvent.summarize( + event = event, + chunkCount = events.count { it.kind == ChunkEvent.KIND }, + ) + }, unreadable = items.size - events.size, awaitsYou = FrostSigningManager.isAwaitingApproval(local.session, items) ) diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/state/UploadReview.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/state/UploadReview.kt index cf817af2..931d5b3d 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/state/UploadReview.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/state/UploadReview.kt @@ -36,7 +36,7 @@ object UploadReview { private fun row(file: PickedFile, ordinal: Int): PickedChapter { val decoded = ChapterText.decode(file.bytes) val markdown = (decoded as? ChapterText.Result.Text)?.markdown.orEmpty() - val chunks = Markdown.splitParagraphs(markdown) + val chunks = Markdown.splitChunks(markdown) return PickedChapter( // Neither the file's name nor its text is unique -- two directories diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterChunkSplitTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterChunkSplitTest.kt index e897939e..303a03c4 100644 --- a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterChunkSplitTest.kt +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterChunkSplitTest.kt @@ -168,7 +168,7 @@ class ChapterChunkSplitTest { val text = "One.\n\nTwo.\n\n\nThree.\n\n \n\nFour." assertEquals( - Markdown.splitParagraphs(text).size, + Markdown.splitChunks(text).size, ChunkEvent.splitOf(unsignedChapter(originalText = text)).size ) } diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposalSizeTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposalSizeTest.kt index dc9c477e..ec1080fe 100644 --- a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposalSizeTest.kt +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposalSizeTest.kt @@ -80,7 +80,7 @@ class ChapterProposalSizeTest { for (paragraphs in shapes) { for (kib in sizes) { val text = markdownOf(kib * 1024, paragraphs) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) val estimated = ChapterProposal.proposalBytes(text, chunks) val actual = encodedBytes(text) @@ -98,7 +98,7 @@ class ChapterProposalSizeTest { // Not merely conservative -- useful. An estimate twice the truth would // pass the test above and refuse half of every legal chapter. val text = markdownOf(16 * 1024, 40) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) val estimated = ChapterProposal.proposalBytes(text, chunks) val actual = encodedBytes(text) @@ -117,7 +117,7 @@ class ChapterProposalSizeTest { val chunkCount = AddChapterMaxChunks val allowance = ChapterProposal.maxOriginalTextBytes(chunkCount) val text = markdownOf(allowance, chunkCount) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) assertEquals(chunkCount, chunks.size) assertTrue(ChapterProposal.fits(text, chunks)) @@ -134,7 +134,7 @@ class ChapterProposalSizeTest { // paragraphs is a long chapter of a real book, not a pathological one -- // and measured, the fit runs out at about 24 KiB for that shape. val text = markdownOf(28 * 1024, 40) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) assertTrue( encodedBytes(text) > FrostSigningManager.MAX_PROPOSAL_BYTES, @@ -162,7 +162,7 @@ class ChapterProposalSizeTest { // The other half of the claim: this is a limit on very long chapters, // not a tax on normal ones. Twelve paragraphs of a few hundred words. val text = markdownOf(8 * 1024, 12) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) assertTrue(ChapterProposal.fits(text, chunks)) assertTrue(encodedBytes(text) < FrostSigningManager.MAX_PROPOSAL_BYTES / 2) diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/ChapterTextTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/ChapterTextTest.kt index 712ee976..aa99eaed 100644 --- a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/ChapterTextTest.kt +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/ChapterTextTest.kt @@ -78,7 +78,7 @@ class ChapterTextTest { fun `splitting is unaffected by which newlines arrived`() { val crlf = decoded("One.\r\n\r\nTwo.\r\n\r\nThree.".encodeToByteArray()) - assertEquals(listOf("One.", "Two.", "Three."), Markdown.splitParagraphs(crlf)) + assertEquals(listOf("One.", "Two.", "Three."), Markdown.splitChunks(crlf)) } @Test diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt new file mode 100644 index 00000000..6b9f38c7 --- /dev/null +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt @@ -0,0 +1,258 @@ +package press.mantra.compose.text + +// GENERATED by docs/scripts/markdown-block-fixtures.js -- do not edit by hand. +// +// The expected output of [Markdown.splitBlocks] for 30 inputs, taken from +// markdown-it with the three departures recorded in docs/chapter-uploads.md +// applied by the generator. Regenerate with: +// +// node docs/scripts/markdown-block-fixtures.js \ +// --markdown-it /node_modules/markdown-it +// +// A change here is a change to what a chunk is, and so to the ids a quorum +// signs. It should arrive with a reason. + +data class Fixture( + val name: String, + val markdown: String, + val chunks: List, +) + +val MARKDOWN_BLOCK_FIXTURES: List = listOf( + Fixture( + name = "one paragraph", + markdown = "A single paragraph.\n", + chunks = listOf( + "A single paragraph.", + ), + ), + Fixture( + name = "two paragraphs", + markdown = "First paragraph.\n\nSecond paragraph.\n", + chunks = listOf( + "First paragraph.", + "Second paragraph.", + ), + ), + Fixture( + name = "lazy continuation", + markdown = "First line.\nSecond line of the same paragraph.\n", + chunks = listOf( + "First line.\nSecond line of the same paragraph.", + ), + ), + Fixture( + name = "atx heading apart", + markdown = "# Chapter one\n\nThe first paragraph.\n", + chunks = listOf( + "Chapter one", + "The first paragraph.", + ), + ), + Fixture( + name = "atx heading glued", + markdown = "# Chapter one\nThe first paragraph.\n", + chunks = listOf( + "Chapter one", + "The first paragraph.", + ), + ), + Fixture( + name = "deep atx heading", + markdown = "#### A fourth-level heading\n\nBody.\n", + chunks = listOf( + "A fourth-level heading", + "Body.", + ), + ), + Fixture( + name = "converter shape", + markdown = "# Chapter one\nFirst paragraph here.\n\nSecond paragraph here.\n", + chunks = listOf( + "Chapter one", + "First paragraph here.", + "Second paragraph here.", + ), + ), + Fixture( + name = "setext h1", + markdown = "Title\n=====\n\nBody.\n", + chunks = listOf( + "Title", + "Body.", + ), + ), + Fixture( + name = "setext h2", + markdown = "Subtitle\n--------\n\nBody.\n", + chunks = listOf( + "Subtitle", + "Body.", + ), + ), + Fixture( + name = "tight bullet list", + markdown = "Shopping:\n\n- milk\n- bread\n- eggs\n", + chunks = listOf( + "Shopping:", + "milk", + "bread", + "eggs", + ), + ), + Fixture( + name = "ordered list", + markdown = "1. first\n2. second\n", + chunks = listOf( + "first", + "second", + ), + ), + Fixture( + name = "nested list", + markdown = "- outer\n - inner\n", + chunks = listOf( + "outer", + "inner", + ), + ), + Fixture( + name = "blockquote two lines", + markdown = "> line one\n> line two\n", + chunks = listOf( + "line one\nline two", + ), + ), + Fixture( + name = "two blockquotes", + markdown = "> first\n\n> second\n", + chunks = listOf( + "first", + "second", + ), + ), + Fixture( + name = "thematic break", + markdown = "One.\n\n---\n\nTwo.\n", + chunks = listOf( + "One.", + "Two.", + ), + ), + Fixture( + name = "fenced code", + markdown = "Text.\n\n```\ncode();\n```\n\nMore.\n", + chunks = listOf( + "Text.", + "More.", + ), + ), + Fixture( + name = "fenced code with info", + markdown = "Text.\n\n```kotlin\nval x = 1\n```\n", + chunks = listOf( + "Text.", + ), + ), + Fixture( + name = "indented code", + markdown = "Text.\n\n indented code\n\nMore.\n", + chunks = listOf( + "Text.", + "More.", + ), + ), + Fixture( + name = "table", + markdown = "| a | b |\n|---|---|\n| 1 | 2 |\n", + chunks = listOf( + "| a | b |\n|---|---|\n| 1 | 2 |", + ), + ), + Fixture( + name = "table with prose", + markdown = "Before.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nAfter.\n", + chunks = listOf( + "Before.", + "| a | b |\n|---|---|\n| 1 | 2 |", + "After.", + ), + ), + Fixture( + name = "link reference definition", + markdown = "See [it][ref].\n\n[ref]: https://example.com\n", + chunks = listOf( + "See [it][ref].", + ), + ), + Fixture( + name = "inline markup survives", + markdown = "A *word* and a [link](https://example.com) and `code`.\n", + chunks = listOf( + "A *word* and a [link](https://example.com) and `code`.", + ), + ), + Fixture( + name = "trailing hard break", + markdown = "Line one \nLine two\n", + chunks = listOf( + "Line one \nLine two", + ), + ), + Fixture( + name = "no trailing newline", + markdown = "No newline at the end.", + chunks = listOf( + "No newline at the end.", + ), + ), + Fixture( + name = "leading blank lines", + markdown = "\n\nA paragraph after blank lines.\n", + chunks = listOf( + "A paragraph after blank lines.", + ), + ), + Fixture( + name = "crlf already normalised", + markdown = "One.\n\nTwo.\n", + chunks = listOf( + "One.", + "Two.", + ), + ), + Fixture( + name = "html block falls back to a paragraph", + markdown = "
html block
\n", + chunks = listOf( + "
html block
", + ), + ), + Fixture( + name = "heading then list then quote", + markdown = "# Title\n\n- one\n- two\n\n> quoted\n\nEnd.\n", + chunks = listOf( + "Title", + "one", + "two", + "quoted", + "End.", + ), + ), + Fixture( + name = "multi paragraph blockquote", + markdown = "> first para\n>\n> second para\n", + chunks = listOf( + "first para", + "second para", + ), + ), + Fixture( + name = "unicode", + markdown = "Ke a leboga.\n\nሰላም ለዓለም።\n", + chunks = listOf( + "Ke a leboga.", + "ሰላም ለዓለም።", + ), + ), +) diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockTest.kt new file mode 100644 index 00000000..d272bad3 --- /dev/null +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockTest.kt @@ -0,0 +1,36 @@ +package press.mantra.compose.text + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * [Markdown.splitBlocks] against markdown-it, which is the parser mantra.press + * translates with. + * + * The fixtures are generated rather than written: `splitBlocks` is hand-rolled, + * its output is hashed into chunk ids a quorum signs, and "it looks right" is + * not a standard that survives a tag being added to an event. See + * `docs/scripts/markdown-block-fixtures.js`, which also encodes the three + * deliberate departures so that this file is a plain equality check. + */ +class MarkdownBlockTest { + + @Test + fun `every fixture splits as markdown-it does`() { + val wrong = MARKDOWN_BLOCK_FIXTURES.mapNotNull { fixture -> + val actual = Markdown.splitChunks(fixture.markdown) + if (actual == fixture.chunks) { + null + } else { + "${fixture.name}:\n expected ${fixture.chunks}\n actual $actual" + } + } + + assertEquals(emptyList(), wrong, "\n" + wrong.joinToString("\n")) + } + + @Test + fun `the corpus is not empty, so a generator that wrote nothing fails`() { + assertEquals(true, MARKDOWN_BLOCK_FIXTURES.size >= 25) + } +} diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownProducerTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownProducerTest.kt new file mode 100644 index 00000000..455ed8a1 --- /dev/null +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownProducerTest.kt @@ -0,0 +1,155 @@ +package press.mantra.compose.text + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Putting a chapter back together with translations in it. + * + * The first test is the one that matters and is the reason stripping block + * markers is safe at all: a chapter with no translations comes back byte for + * byte. Every markup-reassembly bug is then one failing assertion rather than a + * difference somebody has to notice in a rendered document. + */ +class MarkdownProducerTest { + + @Test + fun `every fixture round-trips byte for byte with nothing translated`() { + val wrong = MARKDOWN_BLOCK_FIXTURES.mapNotNull { fixture -> + val out = MarkdownProducer.produce(fixture.markdown, emptyMap()) + if (out == fixture.markdown) null else "${fixture.name}: ${out.encodeToByteArray().size} vs ${fixture.markdown.encodeToByteArray().size}\n got ${out.replace("\n", "\\n")}\n was ${fixture.markdown.replace("\n", "\\n")}" + } + + assertEquals(emptyList(), wrong, "\n" + wrong.joinToString("\n")) + } + + @Test + fun `a translated paragraph replaces its own words`() { + val source = "First paragraph.\n\nSecond paragraph.\n" + + assertEquals( + "First paragraph.\n\nDie tweede paragraaf.\n", + MarkdownProducer.produce(source, mapOf(1 to "Die tweede paragraaf.")), + ) + } + + @Test + fun `a heading keeps its hashes`() { + val source = "# Chapter one\n\nBody.\n" + + assertEquals( + "# Kapitel een\n\nBody.\n", + MarkdownProducer.produce(source, mapOf(0 to "Kapitel een")), + ) + } + + @Test + fun `a heading glued to its paragraph stays glued`() { + // The converter's shape. Under the old blank-line split these were one + // chunk, so the heading could not be translated apart at all. + val source = "# Chapter one\nFirst paragraph.\n" + + assertEquals( + "# Kapitel een\nDie eerste paragraaf.\n", + MarkdownProducer.produce(source, mapOf(0 to "Kapitel een", 1 to "Die eerste paragraaf.")), + ) + } + + @Test + fun `a list item keeps its marker and its indent`() { + val source = "- outer\n - inner\n" + + assertEquals( + "- buitenste\n - binneste\n", + MarkdownProducer.produce(source, mapOf(0 to "buitenste", 1 to "binneste")), + ) + } + + @Test + fun `an ordered list keeps its numbers`() { + val source = "1. first\n2. second\n" + + assertEquals( + "1. eerste\n2. tweede\n", + MarkdownProducer.produce(source, mapOf(0 to "eerste", 1 to "tweede")), + ) + } + + @Test + fun `a blockquote is quoted again, line by line`() { + val source = "> line one\n> line two\n" + + // Two lines in, three out: the marker goes in front of every line, so a + // translation that grew a line is still quoted. + assertEquals( + "> een\n> twee\n> drie\n", + MarkdownProducer.produce(source, mapOf(0 to "een\ntwee\ndrie")), + ) + } + + @Test + fun `a setext heading is underlined to the length of its translation`() { + val source = "Title\n=====\n\nBody.\n" + + assertEquals( + "Titel hier\n==========\n\nBody.\n", + MarkdownProducer.produce(source, mapOf(0 to "Titel hier")), + ) + } + + @Test + fun `code is passed through untranslated and uncounted`() { + val source = "Text.\n\n```\ncode();\n```\n\nMore.\n" + + // The fence carries no chunk, so index 1 is the paragraph after it -- + // which is the whole point of code not being a chunk. + assertEquals( + "Teks.\n\n```\ncode();\n```\n\nMeer.\n", + MarkdownProducer.produce(source, mapOf(0 to "Teks.", 1 to "Meer.")), + ) + } + + @Test + fun `a thematic break and a reference definition are passed through`() { + val source = "One.\n\n---\n\nSee [it][ref].\n\n[ref]: https://example.com\n" + + assertEquals( + "Een.\n\n---\n\nSien [dit][ref].\n\n[ref]: https://example.com\n", + MarkdownProducer.produce(source, mapOf(0 to "Een.", 1 to "Sien [dit][ref].")), + ) + } + + @Test + fun `two identical paragraphs translated differently keep their own words`() { + // The bug in the thing this mirrors. mantra.press matches with + // `translationChunks.find(tc => tc.text == token.content)`, so the first + // translation goes into both. Chunks here carry an index, so position is + // available and exact. + val source = "Same words.\n\nSame words.\n" + + assertEquals( + "Eerste.\n\nTweede.\n", + MarkdownProducer.produce(source, mapOf(0 to "Eerste.", 1 to "Tweede.")), + ) + } + + @Test + fun `an empty translation leaves the original alone`() { + // A translation chunk exists before it has words in it, so this is the + // ordinary case for most of a chapter's life, not an edge one. + val source = "First.\n\nSecond.\n" + + assertEquals(source, MarkdownProducer.produce(source, mapOf(0 to "", 1 to ""))) + } + + @Test + fun `the chunk count agrees with the splitter`() { + MARKDOWN_BLOCK_FIXTURES.forEach { fixture -> + assertEquals( + fixture.chunks.size, + MarkdownProducer.chunkCount(fixture.markdown), + fixture.name, + ) + } + } +} diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/ui/view/state/UploadReviewTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/ui/view/state/UploadReviewTest.kt index 479ed1f3..d910d905 100644 --- a/composeApp/src/commonTest/kotlin/press/mantra/compose/ui/view/state/UploadReviewTest.kt +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/ui/view/state/UploadReviewTest.kt @@ -35,7 +35,10 @@ class UploadReviewTest { assertEquals(1, rows.size) assertNull(rows.single().problem) assertEquals("The Wreck", rows.single().name) - assertEquals(listOf("# The wreck", "It began badly."), rows.single().chunks) + // The heading is its own chunk and carries no `#`: a chunk is a leaf + // block's words with the block marker taken off, so a translator cannot + // break the document by mistyping it. See Markdown.splitBlocks. + assertEquals(listOf("The wreck", "It began badly."), rows.single().chunks) } @Test diff --git a/docs/chapter-uploads.md b/docs/chapter-uploads.md index d5695041..ba1d81cf 100644 --- a/docs/chapter-uploads.md +++ b/docs/chapter-uploads.md @@ -890,6 +890,37 @@ Either take this phase ahead of Phase 2, or treat Phase 2's constant as provisional and re-run its encoded-proposal measurement here. The second is cheaper and the measurement is the same test either way. +**Measured, and it did not move.** `ChapterProposalSizeTest`'s pinned boundary — +24 KiB of markdown over forty paragraphs — is identical before and after, for a +plain reason the warning above missed: prose separated by blank lines splits the +same way under both rules, so the fixture that pins the number is exactly the +shape the change does not touch. What *is* true is the narrower claim: a +list-heavy or heading-heavy chapter now reaches both caps sooner than the same +bytes of plain prose would, which is a real effect on real chapters and shows up +in the form's own counter rather than in a constant. + +### What else came out of it + +Three things the plan had not separated out: + +- **`splitBlocks` returns blocks, not strings.** The marker has to be kept + *somewhere* for the producer, and the chapter's `original_text` is the wrong + place to look it up from — matching text back to its marker is the + string-equality bug this phase fixes elsewhere. So `MarkdownBlock(kind, + markup, text, indent)`, and `splitChunks` for the callers that only want the + words. +- **The scanner segments the whole text, not just the chunks.** + `MarkdownSegment` covers every line, with `block = null` for a fence, a + thematic break, a reference definition or a blank line. That is what makes the + byte-for-byte round trip possible at all: an untranslated chunk is reassembled + from its own source lines rather than re-rendered from its parsed form, so the + round trip is exact by construction rather than by getting every markup rule + right. +- **A blockquote containing a blank `>` line is two chunks, not one.** + markdown-it reads it as two inline runs, so the scanner re-segments it per + inner paragraph — and the fixture for it is the one that would have caught + this being missed. + ### Tests `commonTest`, all of it pure: @@ -900,10 +931,13 @@ cheaper and the measurement is the same test either way. block yield no chunk, a table yields one, a reference definition yields none. - inline markup survives: `*italic*` and `[text](url)` are in the chunk's text. - `MarkdownProducerTest`: round-trip with no translations reproduces the input - byte for byte (the assertion that catches every markup-reassembly bug at - once), and a chapter with two identical paragraphs translated differently - comes back with both translations in the right places — the case mantra.press - gets wrong. + byte for byte **over all thirty fixtures** (the assertion that catches every + markup-reassembly bug at once), and a chapter with two identical paragraphs + translated differently comes back with both translations in the right places — + the case mantra.press gets wrong. Plus the markers going back on: hashes, list + markers and indents, a blockquote re-quoted line by line so a translation that + grew a line is still quoted, and a setext underline re-cut to the length of + the translated title. - `ChapterChunkSplitTest` keeps its existing round-trip and id-stability assertions, now with the converter's output shape as a fixture. diff --git a/docs/scripts/markdown-block-fixtures.js b/docs/scripts/markdown-block-fixtures.js new file mode 100644 index 00000000..ffb008e6 --- /dev/null +++ b/docs/scripts/markdown-block-fixtures.js @@ -0,0 +1,177 @@ +#!/usr/bin/env node +// +// Generates the golden fixtures for `Markdown.splitBlocks`, from markdown-it. +// +// `Markdown.splitBlocks` is hand-rolled, and its output is hashed into chunk +// event ids that a quorum signs -- so it is pinned by measurement against the +// reference implementation rather than by anybody's reading of CommonMark. This +// script is that measurement. It runs markdown-it over the corpus below and +// writes the expected chunks as a Kotlin source file in commonTest. +// +// node docs/scripts/markdown-block-fixtures.js \ +// --markdown-it \ +// [--out composeApp/src/commonTest/kotlin/.../MarkdownBlockFixtures.kt] +// +// mantra.press has one: its node_modules/markdown-it. Pass that path. +// +// THE THREE DEPARTURES are applied here, in the oracle, so the Kotlin test is a +// plain equality check and the departures are visible in one place: +// +// 1. A fenced or indented code block yields no chunk. markdown-it's `fence` +// and `code_block` tokens carry content, so mantra.press turns `code();` +// into a chunk -- with a trailing newline in its text -- and asks a +// translator to translate it and the group to sign it as prose. +// 2. A table is one chunk of its own source, not one per cell. Four chunks +// for a 2x2 table is markdown-it's honest reading, but each costs an event +// envelope to carry one word and a table's alignment is structure that +// per-cell translation cannot preserve. +// 3. A link reference definition yields no chunk. This one is free: it is +// markdown-it's own behaviour with the `reference` block rule left +// enabled. mantra.press disables it -- reaching for the wrong rule name to +// stop links rendering -- which drops `[ref]: https://...` through to +// `paragraph` and makes a translatable chunk of a URL. +// +// `html: false` is markdown-it's default and is kept: an HTML block comes back +// as a paragraph, which is what the Kotlin subset does too. + +const fs = require('fs') +const path = require('path') + +function arg(name, fallback) { + const i = process.argv.indexOf(name) + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback +} + +const mdPath = arg('--markdown-it', null) +if (!mdPath) { + console.error('--markdown-it is required') + process.exit(2) +} +const outPath = arg( + '--out', + 'composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt' +) + +const md = require(path.resolve(mdPath))() + +/** + * The corpus. Each entry is a shape that either appears in real chapters or has + * already caught something. The converter shapes come from mantra.press's own + * scripts/convert-btc-tex-to-markdown.js, which is what the files being + * uploaded are made by. + */ +const CORPUS = { + 'one paragraph': 'A single paragraph.\n', + 'two paragraphs': 'First paragraph.\n\nSecond paragraph.\n', + 'lazy continuation': 'First line.\nSecond line of the same paragraph.\n', + 'atx heading apart': '# Chapter one\n\nThe first paragraph.\n', + 'atx heading glued': '# Chapter one\nThe first paragraph.\n', + 'deep atx heading': '#### A fourth-level heading\n\nBody.\n', + 'converter shape': '# Chapter one\nFirst paragraph here.\n\nSecond paragraph here.\n', + 'setext h1': 'Title\n=====\n\nBody.\n', + 'setext h2': 'Subtitle\n--------\n\nBody.\n', + 'tight bullet list': 'Shopping:\n\n- milk\n- bread\n- eggs\n', + 'ordered list': '1. first\n2. second\n', + 'nested list': '- outer\n - inner\n', + 'blockquote two lines': '> line one\n> line two\n', + 'two blockquotes': '> first\n\n> second\n', + 'thematic break': 'One.\n\n---\n\nTwo.\n', + 'fenced code': 'Text.\n\n```\ncode();\n```\n\nMore.\n', + 'fenced code with info': 'Text.\n\n```kotlin\nval x = 1\n```\n', + 'indented code': 'Text.\n\n indented code\n\nMore.\n', + 'table': '| a | b |\n|---|---|\n| 1 | 2 |\n', + 'table with prose': 'Before.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nAfter.\n', + 'link reference definition': 'See [it][ref].\n\n[ref]: https://example.com\n', + 'inline markup survives': 'A *word* and a [link](https://example.com) and `code`.\n', + 'trailing hard break': 'Line one \nLine two\n', + 'no trailing newline': 'No newline at the end.', + 'leading blank lines': '\n\nA paragraph after blank lines.\n', + 'crlf already normalised': 'One.\n\nTwo.\n', + 'html block falls back to a paragraph': '
html block
\n', + 'heading then list then quote': + '# Title\n\n- one\n- two\n\n> quoted\n\nEnd.\n', + 'multi paragraph blockquote': '> first para\n>\n> second para\n', + 'unicode': 'Ke a leboga.\n\nሰላም ለዓለም።\n', +} + +/** The expected chunks for one input, with the three departures applied. */ +function expected(source) { + const lines = source.split('\n') + const tokens = md.parse(source, {}) + const out = [] + + let tableDepth = 0 + + for (const token of tokens) { + if (token.type === 'table_open') { + // Departure 2: one chunk of the table's own source. + const [start, end] = token.map + out.push(lines.slice(start, end).join('\n')) + tableDepth++ + continue + } + if (token.type === 'table_close') { + tableDepth-- + continue + } + if (tableDepth > 0) continue + + // Departure 1: code carries content and is not prose. + if (token.type === 'fence' || token.type === 'code_block') continue + + // Departure 3 needs no code: with the `reference` rule enabled, a link + // reference definition produces no token at all. + + if (token.type === 'inline' && token.content) out.push(token.content) + } + + return out +} + +function kotlinString(value) { + return '"' + value + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\$/g, '\\$') + .replace(/\n/g, '\\n') + .replace(/\r/g, '\\r') + .replace(/\t/g, '\\t') + '"' +} + +const entries = Object.entries(CORPUS).map(([name, source]) => { + const chunks = expected(source) + return ` Fixture( + name = ${kotlinString(name)}, + markdown = ${kotlinString(source)}, + chunks = listOf(${chunks.length === 0 ? '' : '\n' + chunks.map(c => ' ' + kotlinString(c)).join(',\n') + ',\n '}), + ),` +}).join('\n') + +const file = `package press.mantra.compose.text + +// GENERATED by docs/scripts/markdown-block-fixtures.js -- do not edit by hand. +// +// The expected output of [Markdown.splitBlocks] for ${Object.keys(CORPUS).length} inputs, taken from +// markdown-it with the three departures recorded in docs/chapter-uploads.md +// applied by the generator. Regenerate with: +// +// node docs/scripts/markdown-block-fixtures.js \\ +// --markdown-it /node_modules/markdown-it +// +// A change here is a change to what a chunk is, and so to the ids a quorum +// signs. It should arrive with a reason. + +data class Fixture( + val name: String, + val markdown: String, + val chunks: List, +) + +val MARKDOWN_BLOCK_FIXTURES: List = listOf( +${entries} +) +` + +fs.mkdirSync(path.dirname(outPath), { recursive: true }) +fs.writeFileSync(outPath, file) +console.log(`wrote ${Object.keys(CORPUS).length} fixtures to ${outPath}`)