diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposal.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposal.kt index 0fb718c5..3961d80c 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposal.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposal.kt @@ -66,7 +66,7 @@ object ChapterProposal { (chunks.size + 1) * EVENT_OVERHEAD_BYTES fun proposalBytes(originalText: String): Int = - proposalBytes(originalText, Markdown.splitParagraphs(originalText)) + proposalBytes(originalText, Markdown.splitChunks(originalText)) fun fits(originalText: String, chunks: List): Boolean = proposalBytes(originalText, chunks) <= MAX_PROPOSAL_BYTES diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChunkEvent.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChunkEvent.kt index 48876fc9..9c363b2f 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChunkEvent.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/nip30303/ChunkEvent.kt @@ -63,13 +63,13 @@ class ChunkEvent( sig = chapter.sig ).originalText() ?: return emptyList() - return Markdown.splitParagraphs(originalText).mapIndexed { index, paragraph -> + return Markdown.splitChunks(originalText).mapIndexed { index, chunk -> build( chapterId = chapter.id, - text = paragraph, + text = chunk, index = index, - wordCount = Markdown.wordCount(paragraph), - characterCount = Markdown.characterCount(paragraph), + wordCount = Markdown.wordCount(chunk), + characterCount = Markdown.characterCount(chunk), createdAt = chapter.createdAt, ) } diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/Markdown.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/Markdown.kt index 5771c2dd..30a5cc86 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/Markdown.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/Markdown.kt @@ -9,8 +9,6 @@ package press.mantra.compose.text object Markdown { private val WHITESPACE = Regex("\\s+") - /** A paragraph break is one or more blank lines. */ - private val PARAGRAPH_BREAK = Regex("(?:\\r?\\n){2,}") fun wordCount(text: String): Int { val trimmed = text.trim() @@ -20,11 +18,30 @@ object Markdown { fun characterCount(text: String): Int = text.length /** - * Split markdown into paragraphs (blocks separated by blank lines), - * trimmed and with empties removed. Each paragraph becomes a chunk. + * The blocks a chapter's markdown splits into, one chunk each. + * + * A chunk is a **leaf** block's inline text with its block marker removed: + * the words a translator is given and the words the group signs. Containers + * -- lists, blockquotes -- contribute nothing of their own; a thematic + * break, a code block and a link reference definition contribute nothing at + * all. + * + * This is markdown-it's block token stream, which is what mantra.press + * translates against, with the three departures recorded in + * `docs/chapter-uploads.md`. It replaced a blank-line split, and the two + * disagreed on ten cases in eleven -- on the text as well as the + * boundaries, because markdown-it strips the marker, so `# Chapter one` is + * `"Chapter one"`. Inline markup is kept verbatim: emphasis and links are + * the translator's to carry, where a `#` mistyped into a translation breaks + * the document. + * + * Its output is hashed into chunk ids a quorum signs, so it is pinned by + * `MarkdownBlockFixtures` -- generated from markdown-it itself -- rather + * than by this comment. See [MarkdownBlocks] for the subset it recognises. */ - fun splitParagraphs(text: String): List = - text.split(PARAGRAPH_BREAK) - .map { it.trim() } - .filter { it.isNotEmpty() } + fun splitBlocks(text: String): List = + MarkdownBlocks.segment(text).mapNotNull { it.block } + + /** Just the words of each chunk, which is all an event carries. */ + fun splitChunks(text: String): List = splitBlocks(text).map { it.text } } diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownBlocks.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownBlocks.kt new file mode 100644 index 00000000..99751009 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownBlocks.kt @@ -0,0 +1,337 @@ +package press.mantra.compose.text + +/** + * What kind of leaf block a chunk came out of. + * + * Carried so [MarkdownProducer] can put the block's marker back when the + * chunk's words have been replaced by a translation. Nothing else reads it. + */ +enum class BlockKind { + Paragraph, + Heading, + SetextHeading, + ListItem, + Quote, + + /** + * A whole table, as its own source. + * + * One of the three deliberate departures from markdown-it, which gives a + * chunk per cell. See `docs/chapter-uploads.md`. + */ + Table, +} + +/** + * One chunk, as the words and the marker that were in front of them. + * + * [text] is what a translator is given and what the group signs: the leaf + * block's inline source with the **block** marker removed -- no `#`, no `> `, + * no `- `, no setext underline -- while inline markup (`*italic*`, a link's + * brackets) is kept verbatim. The first is structure a translation should not + * be able to break by mistyping; the second is the translator's to carry. + * + * [markup] is the marker that was removed, kept so a document can be put back + * together. For an ATX heading it is the run of `#`; for a list item the + * bullet or the ordinal; for a blockquote `>`. It is not stored on the chunk + * event -- it is recoverable from the chapter's own `original_text`, which is + * where [MarkdownProducer] reads it from. + */ +data class MarkdownBlock( + val kind: BlockKind, + val markup: String, + val text: String, + /** The indent the block sat at, so a nested list item goes back where it was. */ + val indent: String = "", +) + +/** + * A run of lines, and the chunk it is -- or nothing, if it is not prose. + * + * A segmentation rather than a split, because two things need it. [block] being + * null is a fenced code block, a thematic break, a link reference definition or + * a blank line: material that carries no chunk and still has to be in the + * document afterwards. + * + * [lines] is the exact source, so a document with no translations applied is + * reassembled byte for byte by joining every segment's lines with `\n`. + */ +data class MarkdownSegment( + val lines: List, + val block: MarkdownBlock?, +) + +/** + * The block structure of a chapter's markdown. + * + * A documented **subset** of CommonMark, and hand-rolled rather than taken from + * a library on purpose: its output is hashed into chunk ids that a quorum + * signs, so a dependency bump that moved one boundary would silently re-split + * every chapter proposed afterwards and nothing in the app would notice. What + * pins it instead is `MarkdownBlockFixtures`, generated from markdown-it by + * `docs/scripts/markdown-block-fixtures.js`. + * + * What it recognises: ATX and setext headings, paragraphs with lazy + * continuation, blockquotes, bullet and ordered list items including one level + * of nesting, fenced and indented code, thematic breaks, tables, and link + * reference definitions. + * + * What it does not, each of which has a fixture saying what happens instead: + * HTML blocks (markdown-it's default is `html: false`, so a `
` is a + * paragraph on both sides), footnotes, and blockquotes containing lists. + */ +internal object MarkdownBlocks { + + private val ATX = Regex("^ {0,3}(#{1,6})(?:[ \t]+(.*?))?[ \t]*#*[ \t]*$") + private val SETEXT_UNDERLINE = Regex("^ {0,3}(=+|-+)[ \t]*$") + private val THEMATIC_BREAK = Regex("^ {0,3}((\\*[ \t]*){3,}|(-[ \t]*){3,}|(_[ \t]*){3,})$") + private val FENCE = Regex("^ {0,3}(`{3,}|~{3,})(.*)$") + private val BULLET = Regex("^( {0,3})([-*+])([ \t]+)(.*)$") + private val ORDERED = Regex("^( {0,3})(\\d{1,9}[.)])([ \t]+)(.*)$") + private val QUOTE = Regex("^ {0,3}>[ \t]?(.*)$") + private val REFERENCE_DEFINITION = Regex("^ {0,3}\\[[^\\]]+\\]:[ \t]*\\S.*$") + private val INDENTED_CODE = Regex("^ {4,}\\S.*$") + private val TABLE_DELIMITER = Regex("^ {0,3}\\|?[ \t]*:?-{1,}:?[ \t]*(\\|[ \t]*:?-{1,}:?[ \t]*)*\\|?[ \t]*$") + + fun segment(text: String): List { + val lines = text.split("\n") + val segments = mutableListOf() + var i = 0 + + while (i < lines.size) { + val line = lines[i] + + when { + line.isBlank() -> { + segments += MarkdownSegment(listOf(line), null) + i++ + } + + FENCE.matches(line) -> { + val fence = FENCE.find(line)!!.groupValues[1] + val taken = mutableListOf(lines[i]) + i++ + while (i < lines.size) { + taken += lines[i] + val closing = FENCE.find(lines[i])?.groupValues?.get(1) + i++ + if (closing != null && closing.length >= fence.length) break + } + // Departure 1: code is not prose, so no chunk. + segments += MarkdownSegment(taken, null) + } + + THEMATIC_BREAK.matches(line) -> { + segments += MarkdownSegment(listOf(line), null) + i++ + } + + REFERENCE_DEFINITION.matches(line) -> { + // Departure 3, which is markdown-it's own behaviour with the + // `reference` rule enabled: a definition is not prose. + segments += MarkdownSegment(listOf(line), null) + i++ + } + + INDENTED_CODE.matches(line) -> { + val taken = mutableListOf() + while (i < lines.size && (INDENTED_CODE.matches(lines[i]) || lines[i].isBlank())) { + // A blank line only belongs to the code block if more code + // follows it; otherwise it is the separator after it. + if (lines[i].isBlank() && + lines.drop(i + 1).firstOrNull { it.isNotBlank() } + ?.let { !INDENTED_CODE.matches(it) } != false + ) { + break + } + taken += lines[i] + i++ + } + segments += MarkdownSegment(taken, null) + } + + ATX.matches(line) -> { + val match = ATX.find(line)!! + segments += MarkdownSegment( + lines = listOf(line), + block = MarkdownBlock( + kind = BlockKind.Heading, + markup = match.groupValues[1], + text = match.groupValues[2].trim(), + ), + ) + i++ + } + + isTableStart(lines, i) -> { + val taken = mutableListOf() + while (i < lines.size && lines[i].isNotBlank() && lines[i].contains('|')) { + taken += lines[i] + i++ + } + // Departure 2: one chunk of the whole table's source. + segments += MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.Table, + markup = "", + text = taken.joinToString("\n"), + ), + ) + } + + QUOTE.matches(line) -> { + val taken = mutableListOf() + val inner = mutableListOf() + while (i < lines.size && QUOTE.matches(lines[i])) { + taken += lines[i] + inner += QUOTE.find(lines[i])!!.groupValues[1] + i++ + } + // A `>` on its own splits one blockquote into two paragraphs, + // which markdown-it reads as two inline runs and so two chunks. + val paragraphs = inner.splitOnBlank() + if (paragraphs.size <= 1) { + segments += MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.Quote, + markup = ">", + text = inner.joinToString("\n").trim(), + ), + ) + } else { + // Re-segmented per inner paragraph, so each is its own + // chunk and the source still reassembles exactly. + var consumed = 0 + paragraphs.forEachIndexed { index, paragraph -> + val span = paragraph.size + if (index < paragraphs.size - 1) 1 else 0 + segments += MarkdownSegment( + lines = taken.subList(consumed, consumed + span).toList(), + block = MarkdownBlock( + kind = BlockKind.Quote, + markup = ">", + text = paragraph.joinToString("\n").trim(), + ), + ) + consumed += span + } + } + } + + BULLET.matches(line) || ORDERED.matches(line) -> { + val match = (BULLET.find(line) ?: ORDERED.find(line))!! + val indent = match.groupValues[1] + val marker = match.groupValues[2] + val gap = match.groupValues[3] + val taken = mutableListOf(line) + val content = mutableListOf(match.groupValues[4]) + i++ + // Lazy continuation: a plain line under an item belongs to it. + // A new marker, at any indent, is a new item -- which is what + // makes a nested list two chunks rather than one. + while (i < lines.size && isLazyContinuation(lines[i])) { + taken += lines[i] + content += lines[i].trim() + i++ + } + segments += MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.ListItem, + markup = marker + gap, + text = content.joinToString("\n").trim(), + indent = indent, + ), + ) + } + + else -> { + // A paragraph, unless the line after it underlines it. + val taken = mutableListOf(line) + i++ + var setext: String? = null + while (i < lines.size) { + val next = lines[i] + if (SETEXT_UNDERLINE.matches(next) && !THEMATIC_BREAK.matches(next)) { + setext = SETEXT_UNDERLINE.find(next)!!.groupValues[1] + taken += next + i++ + break + } + if (!isLazyContinuation(next)) break + taken += next + i++ + } + + segments += if (setext != null) { + MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.SetextHeading, + markup = setext, + text = taken.dropLast(1).joinToString("\n").trim(), + ), + ) + } else { + MarkdownSegment( + lines = taken, + block = MarkdownBlock( + kind = BlockKind.Paragraph, + markup = "", + text = taken.joinToString("\n").trim(), + ), + ) + } + } + } + } + + return segments + } + + /** + * Whether [line] continues the block above it rather than starting one. + * + * The list is every block start this scanner recognises. A line that starts + * none of them, and is not blank, is prose belonging to whatever came + * before -- which is what markdown-it calls a lazy continuation and is why + * `First line.\nSecond line.` is one chunk and not two. + */ + private fun isLazyContinuation(line: String): Boolean = when { + line.isBlank() -> false + ATX.matches(line) -> false + THEMATIC_BREAK.matches(line) -> false + FENCE.matches(line) -> false + QUOTE.matches(line) -> false + BULLET.matches(line) -> false + ORDERED.matches(line) -> false + REFERENCE_DEFINITION.matches(line) -> false + else -> true + } + + /** A table is a row of cells with a delimiter row under it. */ + private fun isTableStart(lines: List, at: Int): Boolean = + lines[at].contains('|') && + at + 1 < lines.size && + lines[at + 1].contains('-') && + TABLE_DELIMITER.matches(lines[at + 1]) + + /** Splits a blockquote's inner lines into its paragraphs. */ + private fun List.splitOnBlank(): List> { + val out = mutableListOf>() + var current = mutableListOf() + forEach { line -> + if (line.isBlank()) { + if (current.isNotEmpty()) { + out += current + current = mutableListOf() + } + } else { + current += line + } + } + if (current.isNotEmpty()) out += current + return out + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownProducer.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownProducer.kt new file mode 100644 index 00000000..155345b7 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/MarkdownProducer.kt @@ -0,0 +1,86 @@ +package press.mantra.compose.text + +/** + * A chapter's markdown with translations in place of its words. + * + * The other half of [Markdown.splitBlocks], and the piece that makes stripping + * block markers safe. A chunk carries `"Chapter one"` rather than + * `"# Chapter one"` so that a translator cannot break the document by mistyping + * the `#` -- which is only defensible if something can put the `#` back. Before + * this, nothing in the app could: there was no equivalent of mantra.press's + * `lib/markdown-producer/index.js` at all. + * + * **Substitution is by index, not by text**, and that is strictly better than + * the thing it mirrors. mantra.press matches with + * `translationChunks.find(tc => tc.text == token.content)`, so a chapter with + * two identical paragraphs puts the first one's translation into both of them. + * A chunk here already carries an `IndexTag`, so position is available and + * exact. + */ +object MarkdownProducer { + + /** + * [originalText] rebuilt with [translations] substituted by chunk index. + * + * A chunk with no translation keeps its own source **verbatim** -- not + * re-rendered from its parsed form -- so a chapter with no translations at + * all comes back byte for byte. That is the property worth having: it means + * every markup-reassembly bug is one failing assertion away rather than a + * difference somebody has to notice in a rendered document. + * + * A translated chunk is rendered from its [MarkdownBlock.kind]: the marker + * goes back in front, and for a blockquote in front of every line, so a + * translation that grew a line is still quoted. + */ + fun produce(originalText: String, translations: Map): String { + var chunkIndex = 0 + + return MarkdownBlocks.segment(originalText).flatMap { segment -> + val block = segment.block ?: return@flatMap segment.lines + + val translation = translations[chunkIndex] + chunkIndex++ + + if (translation.isNullOrEmpty()) segment.lines else render(block, translation) + }.joinToString("\n") + } + + /** How many chunks [originalText] has, which is what a caller indexes against. */ + fun chunkCount(originalText: String): Int = Markdown.splitBlocks(originalText).size + + private fun render(block: MarkdownBlock, translation: String): List = + when (block.kind) { + BlockKind.Paragraph -> translation.lines() + + BlockKind.Heading -> listOf("${block.markup} $translation") + + // The underline has to be as long as the text, or it stops being an + // underline -- a one-character `=` under a translated title is a + // setext heading in CommonMark but reads as a typo. + BlockKind.SetextHeading -> { + // The underline matches the line immediately above it, which is + // the last line of the translation rather than its first: a `=` + // shorter than the text it underlines is still a setext heading + // in CommonMark but reads as a typo. + val lines = translation.lines() + lines + block.markup.first().toString() + .repeat(lines.last().length.coerceAtLeast(1)) + } + + BlockKind.ListItem -> { + val lines = translation.lines() + listOf("${block.indent}${block.markup}${lines.first()}") + + // A continuation line is indented under the marker, so the + // item stays one item rather than becoming a paragraph after + // it. + lines.drop(1).map { "${block.indent}${" ".repeat(block.markup.length)}$it" } + } + + BlockKind.Quote -> translation.lines().map { "> $it" } + + // A table's chunk is its own source, so its translation is too: there + // is no marker to put back, and re-deriving the pipes from a + // translated table would be guessing at its columns. + BlockKind.Table -> translation.lines() + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/ProposedEvent.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/ProposedEvent.kt index 7b11f466..a13bcba6 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/text/ProposedEvent.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/text/ProposedEvent.kt @@ -31,7 +31,25 @@ object ProposedEvent { */ data class Summary(val label: String, val detail: String) - fun summarize(event: Event): Summary = when (event.kind) { + /** + * [chunkCount] is how many chunks travel with a chapter in the same batch, + * which only a caller holding the batch can know. + * + * It used to be derived here, by re-splitting the chapter's own text. That + * was wrong twice over. It is a *count of the batch* dressed as a fact about + * the text, so across a splitter change the number a member reads in the + * approval gate disagrees with the chunks they are actually approving -- the + * batch was built by the proposer's splitter and counted by the reader's. + * And it re-split the whole markdown of every chapter on every emission of + * the room's proposal list, which at twenty open proposals is a few hundred + * kilobytes of prose re-scanned to produce a number already sitting in the + * items beside it. + * + * Null where the caller has no batch -- a transcript line, a signed event on + * its own -- and the summary then says nothing about chunks rather than + * guessing. + */ + fun summarize(event: Event, chunkCount: Int? = null): Summary = when (event.kind) { DialectEvent.KIND -> Summary( label = "New dialect", detail = DialectEvent( @@ -74,8 +92,6 @@ object ProposedEvent { // is a whole chapter, so its size stands in for it here. The chunk // count says how many of the batch's other items came out of this // text, which is the rest of what is being signed. - val chunkCount = chapter.originalText()?.let { Markdown.splitParagraphs(it).size } - listOfNotNull( chapter.name(), chapter.wordCount()?.let { "$it words" }, diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/AddChapterScreen.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/AddChapterScreen.kt index 98ee337a..4ff53dc8 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/AddChapterScreen.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/AddChapterScreen.kt @@ -168,8 +168,8 @@ fun AddChapterScreen( val originalText = originalTextFieldState.text.toString() val wordCount = Markdown.wordCount(originalText) val characterCount = Markdown.characterCount(originalText) - val chunks = Markdown.splitParagraphs(originalText) - val paragraphCount = chunks.size + val chunks = Markdown.splitChunks(originalText) + val chunkCount = chunks.size // A chapter hangs off a version, and the group has to be able to // sign; without both there is nothing this screen can propose. @@ -179,7 +179,7 @@ fun AddChapterScreen( // and a session signs a bounded number of events. Past that the // chapter has to be split in two, which is worth saying while there // is still a cursor in the text rather than after a failed propose. - val tooManyChunks = paragraphCount > AddChapterViewModel.MAX_CHUNKS_PER_CHAPTER + val tooManyChunks = chunkCount > AddChapterViewModel.MAX_CHUNKS_PER_CHAPTER // The other cap, and the one that had been enforced nowhere: a // proposal carries the chapter whole and every chunk cut from @@ -191,7 +191,7 @@ fun AddChapterScreen( val canProposeChapter = artifactVersion != null && addChapterUIState.canSign && - paragraphCount > 0 && + chunkCount > 0 && !tooManyChunks && !tooLargeToPropose @@ -268,7 +268,7 @@ fun AddChapterScreen( Text( text = stringResource( Res.string.d_chunks_is_more_than_the_group_can_sign, - paragraphCount, + chunkCount, AddChapterViewModel.MAX_CHUNKS_PER_CHAPTER, ), style = MaterialTheme.typography.bodySmall, @@ -278,7 +278,7 @@ fun AddChapterScreen( Text( text = stringResource( Res.string.this_chapter_is_too_long_to_propose, - paragraphCount, + chunkCount, stringResource(Res.string.s_kib, kibibytes(proposalBytes)), stringResource( Res.string.s_kib, @@ -337,7 +337,7 @@ fun AddChapterScreen( kibibytes(ChapterProposal.MAX_PROPOSAL_BYTES), ), ), - paragraphCount, + chunkCount, AddChapterViewModel.MAX_CHUNKS_PER_CHAPTER, ), style = MaterialTheme.typography.labelMedium, diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/FrostSigningScreen.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/FrostSigningScreen.kt index fbecea82..ec1fb4b9 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/FrostSigningScreen.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/composable/FrostSigningScreen.kt @@ -48,6 +48,7 @@ import press.mantra.compose.database.model.types.FrostSigningStage import press.mantra.compose.managers.FrostSigningManager import press.mantra.compose.repository.ChatRepository import press.mantra.compose.repository.FrostSigningRepository +import press.mantra.compose.nostr.nip30303.ChunkEvent import press.mantra.compose.text.ProposedEvent import press.mantra.compose.ui.composable.widgets.profile.ProfileAvatar import press.mantra.compose.ui.theme.MantraTheme @@ -358,10 +359,14 @@ private fun WhatIsBeingSigned(events: List, expected: Int) { ) } + // The chunk count a chapter reports is a count of this batch, so it + // comes from here rather than from re-reading the chapter's text. + val chunkCount = events.count { it.kind == ChunkEvent.KIND } + events.forEachIndexed { index, event -> if (index > 0) HorizontalDivider() - OneThingBeingSigned(event) + OneThingBeingSigned(event, chunkCount) } Text( @@ -376,8 +381,8 @@ private fun WhatIsBeingSigned(events: List, expected: Int) { /** One event of the batch, described as the thing it is. */ @Composable -private fun OneThingBeingSigned(event: Event) { - val summary = ProposedEvent.summarize(event) +private fun OneThingBeingSigned(event: Event, chunkCount: Int) { + val summary = ProposedEvent.summarize(event, chunkCount) Column(verticalArrangement = Arrangement.spacedBy(MaterialTheme.spacing.space50)) { Text(text = summary.label, style = MaterialTheme.typography.labelMedium) diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/AddChapterViewModel.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/AddChapterViewModel.kt index bea91925..8bde3330 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/AddChapterViewModel.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/AddChapterViewModel.kt @@ -102,7 +102,7 @@ class AddChapterViewModel( ) { val name = nameField.text.toString() val originalText = originalTextField.text.toString() - val paragraphs = Markdown.splitParagraphs(originalText) + val chunks = Markdown.splitChunks(originalText) // Both caps are the group's, not this screen's, and proposing past // either throws rather than failing softly. The form says so as it is @@ -113,9 +113,9 @@ class AddChapterViewModel( // weighs. A chapter of sixty-three one-word paragraphs passes the second // and fails the first; one of three very long ones does the reverse. if (name.isBlank() || - paragraphs.isEmpty() || - paragraphs.size > MAX_CHUNKS_PER_CHAPTER || - !ChapterProposal.fits(originalText, paragraphs) + chunks.isEmpty() || + chunks.size > MAX_CHUNKS_PER_CHAPTER || + !ChapterProposal.fits(originalText, chunks) ) { onFailure.invoke() return diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/ProposalListViewModel.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/ProposalListViewModel.kt index 938d3d83..e6e40587 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/ProposalListViewModel.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/model/ProposalListViewModel.kt @@ -18,6 +18,7 @@ import press.mantra.compose.extensions.shortened import press.mantra.compose.managers.FrostSigningManager import press.mantra.compose.repository.ChatRepository import press.mantra.compose.repository.FrostSigningRepository +import press.mantra.compose.nostr.nip30303.ChunkEvent import press.mantra.compose.text.ProposedEvent import press.mantra.compose.ui.view.state.ProposalListUIState @@ -86,7 +87,15 @@ class ProposalListViewModel( else -> local.proposer?.humanReadableNameOrPubkey() ?: local.session.coordinatorPublicKey.shortened() }, - signs = events.map { ProposedEvent.summarize(it) }, + // Counted from the batch rather than derived from the chapter's + // text -- the number a member reads has to be the number of events + // they are approving. See ProposedEvent.summarize. + signs = events.map { event -> + ProposedEvent.summarize( + event = event, + chunkCount = events.count { it.kind == ChunkEvent.KIND }, + ) + }, unreadable = items.size - events.size, awaitsYou = FrostSigningManager.isAwaitingApproval(local.session, items) ) diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/state/UploadReview.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/state/UploadReview.kt index cf817af2..931d5b3d 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/state/UploadReview.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/ui/view/state/UploadReview.kt @@ -36,7 +36,7 @@ object UploadReview { private fun row(file: PickedFile, ordinal: Int): PickedChapter { val decoded = ChapterText.decode(file.bytes) val markdown = (decoded as? ChapterText.Result.Text)?.markdown.orEmpty() - val chunks = Markdown.splitParagraphs(markdown) + val chunks = Markdown.splitChunks(markdown) return PickedChapter( // Neither the file's name nor its text is unique -- two directories diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterChunkSplitTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterChunkSplitTest.kt index e897939e..303a03c4 100644 --- a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterChunkSplitTest.kt +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterChunkSplitTest.kt @@ -168,7 +168,7 @@ class ChapterChunkSplitTest { val text = "One.\n\nTwo.\n\n\nThree.\n\n \n\nFour." assertEquals( - Markdown.splitParagraphs(text).size, + Markdown.splitChunks(text).size, ChunkEvent.splitOf(unsignedChapter(originalText = text)).size ) } diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposalSizeTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposalSizeTest.kt index dc9c477e..ec1080fe 100644 --- a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposalSizeTest.kt +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/nip30303/ChapterProposalSizeTest.kt @@ -80,7 +80,7 @@ class ChapterProposalSizeTest { for (paragraphs in shapes) { for (kib in sizes) { val text = markdownOf(kib * 1024, paragraphs) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) val estimated = ChapterProposal.proposalBytes(text, chunks) val actual = encodedBytes(text) @@ -98,7 +98,7 @@ class ChapterProposalSizeTest { // Not merely conservative -- useful. An estimate twice the truth would // pass the test above and refuse half of every legal chapter. val text = markdownOf(16 * 1024, 40) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) val estimated = ChapterProposal.proposalBytes(text, chunks) val actual = encodedBytes(text) @@ -117,7 +117,7 @@ class ChapterProposalSizeTest { val chunkCount = AddChapterMaxChunks val allowance = ChapterProposal.maxOriginalTextBytes(chunkCount) val text = markdownOf(allowance, chunkCount) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) assertEquals(chunkCount, chunks.size) assertTrue(ChapterProposal.fits(text, chunks)) @@ -134,7 +134,7 @@ class ChapterProposalSizeTest { // paragraphs is a long chapter of a real book, not a pathological one -- // and measured, the fit runs out at about 24 KiB for that shape. val text = markdownOf(28 * 1024, 40) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) assertTrue( encodedBytes(text) > FrostSigningManager.MAX_PROPOSAL_BYTES, @@ -162,7 +162,7 @@ class ChapterProposalSizeTest { // The other half of the claim: this is a limit on very long chapters, // not a tax on normal ones. Twelve paragraphs of a few hundred words. val text = markdownOf(8 * 1024, 12) - val chunks = Markdown.splitParagraphs(text) + val chunks = Markdown.splitChunks(text) assertTrue(ChapterProposal.fits(text, chunks)) assertTrue(encodedBytes(text) < FrostSigningManager.MAX_PROPOSAL_BYTES / 2) diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/ChapterTextTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/ChapterTextTest.kt index 712ee976..aa99eaed 100644 --- a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/ChapterTextTest.kt +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/ChapterTextTest.kt @@ -78,7 +78,7 @@ class ChapterTextTest { fun `splitting is unaffected by which newlines arrived`() { val crlf = decoded("One.\r\n\r\nTwo.\r\n\r\nThree.".encodeToByteArray()) - assertEquals(listOf("One.", "Two.", "Three."), Markdown.splitParagraphs(crlf)) + assertEquals(listOf("One.", "Two.", "Three."), Markdown.splitChunks(crlf)) } @Test diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt new file mode 100644 index 00000000..6b9f38c7 --- /dev/null +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt @@ -0,0 +1,258 @@ +package press.mantra.compose.text + +// GENERATED by docs/scripts/markdown-block-fixtures.js -- do not edit by hand. +// +// The expected output of [Markdown.splitBlocks] for 30 inputs, taken from +// markdown-it with the three departures recorded in docs/chapter-uploads.md +// applied by the generator. Regenerate with: +// +// node docs/scripts/markdown-block-fixtures.js \ +// --markdown-it /node_modules/markdown-it +// +// A change here is a change to what a chunk is, and so to the ids a quorum +// signs. It should arrive with a reason. + +data class Fixture( + val name: String, + val markdown: String, + val chunks: List, +) + +val MARKDOWN_BLOCK_FIXTURES: List = listOf( + Fixture( + name = "one paragraph", + markdown = "A single paragraph.\n", + chunks = listOf( + "A single paragraph.", + ), + ), + Fixture( + name = "two paragraphs", + markdown = "First paragraph.\n\nSecond paragraph.\n", + chunks = listOf( + "First paragraph.", + "Second paragraph.", + ), + ), + Fixture( + name = "lazy continuation", + markdown = "First line.\nSecond line of the same paragraph.\n", + chunks = listOf( + "First line.\nSecond line of the same paragraph.", + ), + ), + Fixture( + name = "atx heading apart", + markdown = "# Chapter one\n\nThe first paragraph.\n", + chunks = listOf( + "Chapter one", + "The first paragraph.", + ), + ), + Fixture( + name = "atx heading glued", + markdown = "# Chapter one\nThe first paragraph.\n", + chunks = listOf( + "Chapter one", + "The first paragraph.", + ), + ), + Fixture( + name = "deep atx heading", + markdown = "#### A fourth-level heading\n\nBody.\n", + chunks = listOf( + "A fourth-level heading", + "Body.", + ), + ), + Fixture( + name = "converter shape", + markdown = "# Chapter one\nFirst paragraph here.\n\nSecond paragraph here.\n", + chunks = listOf( + "Chapter one", + "First paragraph here.", + "Second paragraph here.", + ), + ), + Fixture( + name = "setext h1", + markdown = "Title\n=====\n\nBody.\n", + chunks = listOf( + "Title", + "Body.", + ), + ), + Fixture( + name = "setext h2", + markdown = "Subtitle\n--------\n\nBody.\n", + chunks = listOf( + "Subtitle", + "Body.", + ), + ), + Fixture( + name = "tight bullet list", + markdown = "Shopping:\n\n- milk\n- bread\n- eggs\n", + chunks = listOf( + "Shopping:", + "milk", + "bread", + "eggs", + ), + ), + Fixture( + name = "ordered list", + markdown = "1. first\n2. second\n", + chunks = listOf( + "first", + "second", + ), + ), + Fixture( + name = "nested list", + markdown = "- outer\n - inner\n", + chunks = listOf( + "outer", + "inner", + ), + ), + Fixture( + name = "blockquote two lines", + markdown = "> line one\n> line two\n", + chunks = listOf( + "line one\nline two", + ), + ), + Fixture( + name = "two blockquotes", + markdown = "> first\n\n> second\n", + chunks = listOf( + "first", + "second", + ), + ), + Fixture( + name = "thematic break", + markdown = "One.\n\n---\n\nTwo.\n", + chunks = listOf( + "One.", + "Two.", + ), + ), + Fixture( + name = "fenced code", + markdown = "Text.\n\n```\ncode();\n```\n\nMore.\n", + chunks = listOf( + "Text.", + "More.", + ), + ), + Fixture( + name = "fenced code with info", + markdown = "Text.\n\n```kotlin\nval x = 1\n```\n", + chunks = listOf( + "Text.", + ), + ), + Fixture( + name = "indented code", + markdown = "Text.\n\n indented code\n\nMore.\n", + chunks = listOf( + "Text.", + "More.", + ), + ), + Fixture( + name = "table", + markdown = "| a | b |\n|---|---|\n| 1 | 2 |\n", + chunks = listOf( + "| a | b |\n|---|---|\n| 1 | 2 |", + ), + ), + Fixture( + name = "table with prose", + markdown = "Before.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nAfter.\n", + chunks = listOf( + "Before.", + "| a | b |\n|---|---|\n| 1 | 2 |", + "After.", + ), + ), + Fixture( + name = "link reference definition", + markdown = "See [it][ref].\n\n[ref]: https://example.com\n", + chunks = listOf( + "See [it][ref].", + ), + ), + Fixture( + name = "inline markup survives", + markdown = "A *word* and a [link](https://example.com) and `code`.\n", + chunks = listOf( + "A *word* and a [link](https://example.com) and `code`.", + ), + ), + Fixture( + name = "trailing hard break", + markdown = "Line one \nLine two\n", + chunks = listOf( + "Line one \nLine two", + ), + ), + Fixture( + name = "no trailing newline", + markdown = "No newline at the end.", + chunks = listOf( + "No newline at the end.", + ), + ), + Fixture( + name = "leading blank lines", + markdown = "\n\nA paragraph after blank lines.\n", + chunks = listOf( + "A paragraph after blank lines.", + ), + ), + Fixture( + name = "crlf already normalised", + markdown = "One.\n\nTwo.\n", + chunks = listOf( + "One.", + "Two.", + ), + ), + Fixture( + name = "html block falls back to a paragraph", + markdown = "
html block
\n", + chunks = listOf( + "
html block
", + ), + ), + Fixture( + name = "heading then list then quote", + markdown = "# Title\n\n- one\n- two\n\n> quoted\n\nEnd.\n", + chunks = listOf( + "Title", + "one", + "two", + "quoted", + "End.", + ), + ), + Fixture( + name = "multi paragraph blockquote", + markdown = "> first para\n>\n> second para\n", + chunks = listOf( + "first para", + "second para", + ), + ), + Fixture( + name = "unicode", + markdown = "Ke a leboga.\n\nሰላም ለዓለም።\n", + chunks = listOf( + "Ke a leboga.", + "ሰላም ለዓለም።", + ), + ), +) diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockTest.kt new file mode 100644 index 00000000..d272bad3 --- /dev/null +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockTest.kt @@ -0,0 +1,36 @@ +package press.mantra.compose.text + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * [Markdown.splitBlocks] against markdown-it, which is the parser mantra.press + * translates with. + * + * The fixtures are generated rather than written: `splitBlocks` is hand-rolled, + * its output is hashed into chunk ids a quorum signs, and "it looks right" is + * not a standard that survives a tag being added to an event. See + * `docs/scripts/markdown-block-fixtures.js`, which also encodes the three + * deliberate departures so that this file is a plain equality check. + */ +class MarkdownBlockTest { + + @Test + fun `every fixture splits as markdown-it does`() { + val wrong = MARKDOWN_BLOCK_FIXTURES.mapNotNull { fixture -> + val actual = Markdown.splitChunks(fixture.markdown) + if (actual == fixture.chunks) { + null + } else { + "${fixture.name}:\n expected ${fixture.chunks}\n actual $actual" + } + } + + assertEquals(emptyList(), wrong, "\n" + wrong.joinToString("\n")) + } + + @Test + fun `the corpus is not empty, so a generator that wrote nothing fails`() { + assertEquals(true, MARKDOWN_BLOCK_FIXTURES.size >= 25) + } +} diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownProducerTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownProducerTest.kt new file mode 100644 index 00000000..455ed8a1 --- /dev/null +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownProducerTest.kt @@ -0,0 +1,155 @@ +package press.mantra.compose.text + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Putting a chapter back together with translations in it. + * + * The first test is the one that matters and is the reason stripping block + * markers is safe at all: a chapter with no translations comes back byte for + * byte. Every markup-reassembly bug is then one failing assertion rather than a + * difference somebody has to notice in a rendered document. + */ +class MarkdownProducerTest { + + @Test + fun `every fixture round-trips byte for byte with nothing translated`() { + val wrong = MARKDOWN_BLOCK_FIXTURES.mapNotNull { fixture -> + val out = MarkdownProducer.produce(fixture.markdown, emptyMap()) + if (out == fixture.markdown) null else "${fixture.name}: ${out.encodeToByteArray().size} vs ${fixture.markdown.encodeToByteArray().size}\n got ${out.replace("\n", "\\n")}\n was ${fixture.markdown.replace("\n", "\\n")}" + } + + assertEquals(emptyList(), wrong, "\n" + wrong.joinToString("\n")) + } + + @Test + fun `a translated paragraph replaces its own words`() { + val source = "First paragraph.\n\nSecond paragraph.\n" + + assertEquals( + "First paragraph.\n\nDie tweede paragraaf.\n", + MarkdownProducer.produce(source, mapOf(1 to "Die tweede paragraaf.")), + ) + } + + @Test + fun `a heading keeps its hashes`() { + val source = "# Chapter one\n\nBody.\n" + + assertEquals( + "# Kapitel een\n\nBody.\n", + MarkdownProducer.produce(source, mapOf(0 to "Kapitel een")), + ) + } + + @Test + fun `a heading glued to its paragraph stays glued`() { + // The converter's shape. Under the old blank-line split these were one + // chunk, so the heading could not be translated apart at all. + val source = "# Chapter one\nFirst paragraph.\n" + + assertEquals( + "# Kapitel een\nDie eerste paragraaf.\n", + MarkdownProducer.produce(source, mapOf(0 to "Kapitel een", 1 to "Die eerste paragraaf.")), + ) + } + + @Test + fun `a list item keeps its marker and its indent`() { + val source = "- outer\n - inner\n" + + assertEquals( + "- buitenste\n - binneste\n", + MarkdownProducer.produce(source, mapOf(0 to "buitenste", 1 to "binneste")), + ) + } + + @Test + fun `an ordered list keeps its numbers`() { + val source = "1. first\n2. second\n" + + assertEquals( + "1. eerste\n2. tweede\n", + MarkdownProducer.produce(source, mapOf(0 to "eerste", 1 to "tweede")), + ) + } + + @Test + fun `a blockquote is quoted again, line by line`() { + val source = "> line one\n> line two\n" + + // Two lines in, three out: the marker goes in front of every line, so a + // translation that grew a line is still quoted. + assertEquals( + "> een\n> twee\n> drie\n", + MarkdownProducer.produce(source, mapOf(0 to "een\ntwee\ndrie")), + ) + } + + @Test + fun `a setext heading is underlined to the length of its translation`() { + val source = "Title\n=====\n\nBody.\n" + + assertEquals( + "Titel hier\n==========\n\nBody.\n", + MarkdownProducer.produce(source, mapOf(0 to "Titel hier")), + ) + } + + @Test + fun `code is passed through untranslated and uncounted`() { + val source = "Text.\n\n```\ncode();\n```\n\nMore.\n" + + // The fence carries no chunk, so index 1 is the paragraph after it -- + // which is the whole point of code not being a chunk. + assertEquals( + "Teks.\n\n```\ncode();\n```\n\nMeer.\n", + MarkdownProducer.produce(source, mapOf(0 to "Teks.", 1 to "Meer.")), + ) + } + + @Test + fun `a thematic break and a reference definition are passed through`() { + val source = "One.\n\n---\n\nSee [it][ref].\n\n[ref]: https://example.com\n" + + assertEquals( + "Een.\n\n---\n\nSien [dit][ref].\n\n[ref]: https://example.com\n", + MarkdownProducer.produce(source, mapOf(0 to "Een.", 1 to "Sien [dit][ref].")), + ) + } + + @Test + fun `two identical paragraphs translated differently keep their own words`() { + // The bug in the thing this mirrors. mantra.press matches with + // `translationChunks.find(tc => tc.text == token.content)`, so the first + // translation goes into both. Chunks here carry an index, so position is + // available and exact. + val source = "Same words.\n\nSame words.\n" + + assertEquals( + "Eerste.\n\nTweede.\n", + MarkdownProducer.produce(source, mapOf(0 to "Eerste.", 1 to "Tweede.")), + ) + } + + @Test + fun `an empty translation leaves the original alone`() { + // A translation chunk exists before it has words in it, so this is the + // ordinary case for most of a chapter's life, not an edge one. + val source = "First.\n\nSecond.\n" + + assertEquals(source, MarkdownProducer.produce(source, mapOf(0 to "", 1 to ""))) + } + + @Test + fun `the chunk count agrees with the splitter`() { + MARKDOWN_BLOCK_FIXTURES.forEach { fixture -> + assertEquals( + fixture.chunks.size, + MarkdownProducer.chunkCount(fixture.markdown), + fixture.name, + ) + } + } +} diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/ui/view/state/UploadReviewTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/ui/view/state/UploadReviewTest.kt index 479ed1f3..d910d905 100644 --- a/composeApp/src/commonTest/kotlin/press/mantra/compose/ui/view/state/UploadReviewTest.kt +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/ui/view/state/UploadReviewTest.kt @@ -35,7 +35,10 @@ class UploadReviewTest { assertEquals(1, rows.size) assertNull(rows.single().problem) assertEquals("The Wreck", rows.single().name) - assertEquals(listOf("# The wreck", "It began badly."), rows.single().chunks) + // The heading is its own chunk and carries no `#`: a chunk is a leaf + // block's words with the block marker taken off, so a translator cannot + // break the document by mistyping it. See Markdown.splitBlocks. + assertEquals(listOf("The wreck", "It began badly."), rows.single().chunks) } @Test diff --git a/docs/chapter-uploads.md b/docs/chapter-uploads.md index d5695041..ba1d81cf 100644 --- a/docs/chapter-uploads.md +++ b/docs/chapter-uploads.md @@ -890,6 +890,37 @@ Either take this phase ahead of Phase 2, or treat Phase 2's constant as provisional and re-run its encoded-proposal measurement here. The second is cheaper and the measurement is the same test either way. +**Measured, and it did not move.** `ChapterProposalSizeTest`'s pinned boundary — +24 KiB of markdown over forty paragraphs — is identical before and after, for a +plain reason the warning above missed: prose separated by blank lines splits the +same way under both rules, so the fixture that pins the number is exactly the +shape the change does not touch. What *is* true is the narrower claim: a +list-heavy or heading-heavy chapter now reaches both caps sooner than the same +bytes of plain prose would, which is a real effect on real chapters and shows up +in the form's own counter rather than in a constant. + +### What else came out of it + +Three things the plan had not separated out: + +- **`splitBlocks` returns blocks, not strings.** The marker has to be kept + *somewhere* for the producer, and the chapter's `original_text` is the wrong + place to look it up from — matching text back to its marker is the + string-equality bug this phase fixes elsewhere. So `MarkdownBlock(kind, + markup, text, indent)`, and `splitChunks` for the callers that only want the + words. +- **The scanner segments the whole text, not just the chunks.** + `MarkdownSegment` covers every line, with `block = null` for a fence, a + thematic break, a reference definition or a blank line. That is what makes the + byte-for-byte round trip possible at all: an untranslated chunk is reassembled + from its own source lines rather than re-rendered from its parsed form, so the + round trip is exact by construction rather than by getting every markup rule + right. +- **A blockquote containing a blank `>` line is two chunks, not one.** + markdown-it reads it as two inline runs, so the scanner re-segments it per + inner paragraph — and the fixture for it is the one that would have caught + this being missed. + ### Tests `commonTest`, all of it pure: @@ -900,10 +931,13 @@ cheaper and the measurement is the same test either way. block yield no chunk, a table yields one, a reference definition yields none. - inline markup survives: `*italic*` and `[text](url)` are in the chunk's text. - `MarkdownProducerTest`: round-trip with no translations reproduces the input - byte for byte (the assertion that catches every markup-reassembly bug at - once), and a chapter with two identical paragraphs translated differently - comes back with both translations in the right places — the case mantra.press - gets wrong. + byte for byte **over all thirty fixtures** (the assertion that catches every + markup-reassembly bug at once), and a chapter with two identical paragraphs + translated differently comes back with both translations in the right places — + the case mantra.press gets wrong. Plus the markers going back on: hashes, list + markers and indents, a blockquote re-quoted line by line so a translation that + grew a line is still quoted, and a setext underline re-cut to the length of + the translated title. - `ChapterChunkSplitTest` keeps its existing round-trip and id-stability assertions, now with the converter's output shape as a fixture. diff --git a/docs/scripts/markdown-block-fixtures.js b/docs/scripts/markdown-block-fixtures.js new file mode 100644 index 00000000..ffb008e6 --- /dev/null +++ b/docs/scripts/markdown-block-fixtures.js @@ -0,0 +1,177 @@ +#!/usr/bin/env node +// +// Generates the golden fixtures for `Markdown.splitBlocks`, from markdown-it. +// +// `Markdown.splitBlocks` is hand-rolled, and its output is hashed into chunk +// event ids that a quorum signs -- so it is pinned by measurement against the +// reference implementation rather than by anybody's reading of CommonMark. This +// script is that measurement. It runs markdown-it over the corpus below and +// writes the expected chunks as a Kotlin source file in commonTest. +// +// node docs/scripts/markdown-block-fixtures.js \ +// --markdown-it \ +// [--out composeApp/src/commonTest/kotlin/.../MarkdownBlockFixtures.kt] +// +// mantra.press has one: its node_modules/markdown-it. Pass that path. +// +// THE THREE DEPARTURES are applied here, in the oracle, so the Kotlin test is a +// plain equality check and the departures are visible in one place: +// +// 1. A fenced or indented code block yields no chunk. markdown-it's `fence` +// and `code_block` tokens carry content, so mantra.press turns `code();` +// into a chunk -- with a trailing newline in its text -- and asks a +// translator to translate it and the group to sign it as prose. +// 2. A table is one chunk of its own source, not one per cell. Four chunks +// for a 2x2 table is markdown-it's honest reading, but each costs an event +// envelope to carry one word and a table's alignment is structure that +// per-cell translation cannot preserve. +// 3. A link reference definition yields no chunk. This one is free: it is +// markdown-it's own behaviour with the `reference` block rule left +// enabled. mantra.press disables it -- reaching for the wrong rule name to +// stop links rendering -- which drops `[ref]: https://...` through to +// `paragraph` and makes a translatable chunk of a URL. +// +// `html: false` is markdown-it's default and is kept: an HTML block comes back +// as a paragraph, which is what the Kotlin subset does too. + +const fs = require('fs') +const path = require('path') + +function arg(name, fallback) { + const i = process.argv.indexOf(name) + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback +} + +const mdPath = arg('--markdown-it', null) +if (!mdPath) { + console.error('--markdown-it is required') + process.exit(2) +} +const outPath = arg( + '--out', + 'composeApp/src/commonTest/kotlin/press/mantra/compose/text/MarkdownBlockFixtures.kt' +) + +const md = require(path.resolve(mdPath))() + +/** + * The corpus. Each entry is a shape that either appears in real chapters or has + * already caught something. The converter shapes come from mantra.press's own + * scripts/convert-btc-tex-to-markdown.js, which is what the files being + * uploaded are made by. + */ +const CORPUS = { + 'one paragraph': 'A single paragraph.\n', + 'two paragraphs': 'First paragraph.\n\nSecond paragraph.\n', + 'lazy continuation': 'First line.\nSecond line of the same paragraph.\n', + 'atx heading apart': '# Chapter one\n\nThe first paragraph.\n', + 'atx heading glued': '# Chapter one\nThe first paragraph.\n', + 'deep atx heading': '#### A fourth-level heading\n\nBody.\n', + 'converter shape': '# Chapter one\nFirst paragraph here.\n\nSecond paragraph here.\n', + 'setext h1': 'Title\n=====\n\nBody.\n', + 'setext h2': 'Subtitle\n--------\n\nBody.\n', + 'tight bullet list': 'Shopping:\n\n- milk\n- bread\n- eggs\n', + 'ordered list': '1. first\n2. second\n', + 'nested list': '- outer\n - inner\n', + 'blockquote two lines': '> line one\n> line two\n', + 'two blockquotes': '> first\n\n> second\n', + 'thematic break': 'One.\n\n---\n\nTwo.\n', + 'fenced code': 'Text.\n\n```\ncode();\n```\n\nMore.\n', + 'fenced code with info': 'Text.\n\n```kotlin\nval x = 1\n```\n', + 'indented code': 'Text.\n\n indented code\n\nMore.\n', + 'table': '| a | b |\n|---|---|\n| 1 | 2 |\n', + 'table with prose': 'Before.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nAfter.\n', + 'link reference definition': 'See [it][ref].\n\n[ref]: https://example.com\n', + 'inline markup survives': 'A *word* and a [link](https://example.com) and `code`.\n', + 'trailing hard break': 'Line one \nLine two\n', + 'no trailing newline': 'No newline at the end.', + 'leading blank lines': '\n\nA paragraph after blank lines.\n', + 'crlf already normalised': 'One.\n\nTwo.\n', + 'html block falls back to a paragraph': '
html block
\n', + 'heading then list then quote': + '# Title\n\n- one\n- two\n\n> quoted\n\nEnd.\n', + 'multi paragraph blockquote': '> first para\n>\n> second para\n', + 'unicode': 'Ke a leboga.\n\nሰላም ለዓለም።\n', +} + +/** The expected chunks for one input, with the three departures applied. */ +function expected(source) { + const lines = source.split('\n') + const tokens = md.parse(source, {}) + const out = [] + + let tableDepth = 0 + + for (const token of tokens) { + if (token.type === 'table_open') { + // Departure 2: one chunk of the table's own source. + const [start, end] = token.map + out.push(lines.slice(start, end).join('\n')) + tableDepth++ + continue + } + if (token.type === 'table_close') { + tableDepth-- + continue + } + if (tableDepth > 0) continue + + // Departure 1: code carries content and is not prose. + if (token.type === 'fence' || token.type === 'code_block') continue + + // Departure 3 needs no code: with the `reference` rule enabled, a link + // reference definition produces no token at all. + + if (token.type === 'inline' && token.content) out.push(token.content) + } + + return out +} + +function kotlinString(value) { + return '"' + value + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\$/g, '\\$') + .replace(/\n/g, '\\n') + .replace(/\r/g, '\\r') + .replace(/\t/g, '\\t') + '"' +} + +const entries = Object.entries(CORPUS).map(([name, source]) => { + const chunks = expected(source) + return ` Fixture( + name = ${kotlinString(name)}, + markdown = ${kotlinString(source)}, + chunks = listOf(${chunks.length === 0 ? '' : '\n' + chunks.map(c => ' ' + kotlinString(c)).join(',\n') + ',\n '}), + ),` +}).join('\n') + +const file = `package press.mantra.compose.text + +// GENERATED by docs/scripts/markdown-block-fixtures.js -- do not edit by hand. +// +// The expected output of [Markdown.splitBlocks] for ${Object.keys(CORPUS).length} inputs, taken from +// markdown-it with the three departures recorded in docs/chapter-uploads.md +// applied by the generator. Regenerate with: +// +// node docs/scripts/markdown-block-fixtures.js \\ +// --markdown-it /node_modules/markdown-it +// +// A change here is a change to what a chunk is, and so to the ids a quorum +// signs. It should arrive with a reason. + +data class Fixture( + val name: String, + val markdown: String, + val chunks: List, +) + +val MARKDOWN_BLOCK_FIXTURES: List = listOf( +${entries} +) +` + +fs.mkdirSync(path.dirname(outPath), { recursive: true }) +fs.writeFileSync(outPath, file) +console.log(`wrote ${Object.keys(CORPUS).length} fixtures to ${outPath}`)