More mantra things

.gitignore

@@ -81,3 +81,4 @@ config/production.json
 database.sqlite
 .vscode/
 .DS_Store
+result/

config/default.json

@@ -15,7 +15,6 @@
         "secret" : "don't forget to keep your secrets secret",
         "port" : 7878,
         "cookie": {
-            // "domain": "mantra.press"
         },
         "compiled-render": false,
         "domain": "https://mantra.press"

package.json

@@ -21,10 +21,10 @@
     "body-parser": "^1.19.1",
     "config": "^3.3.6",
     "connect-session-sequelize": "^7.1.2",
-    "excel4node": "^1.7.2",
     "express": "^4.17.2",
     "express-fileupload": "^1.4.0",
     "express-session": "^1.17.2",
+    "fast-xml-parser": "^4.1.3",
     "mantra-db-models": "git+https://code.sigidli.com/mantra/mantra-db-models.git",
     "markdown-it": "^13.0.1",
     "passport": "^0.5.2",

scripts/convert-btc-tex-to-markdown.js

@@ -0,0 +1,133 @@
+const fs = require('fs');
+const path = require("path")
+const { XMLParser } = require("fast-xml-parser")
+
+const appendOptionalSpace = (nextObject) => {
+    const punctuation = [".", ",", "?", ":", "!"]
+    if (nextObject)  {
+        
+        const nextString = produceString(nextObject)
+
+        
+        if (punctuation.includes(nextString.trim().at(0))) {
+            return ""
+        } 
+        if (nextString.startsWith("—")) {
+            return ""
+        }
+        return " "
+    }
+
+    return ""
+}
+const produceString = (xmlObject, references = [], nextObject = null, level = 1) => {
+    if (Array.isArray(xmlObject)) {
+        const array = xmlObject.map((object, index, array) => {
+            return produceString(object, references, array[index+1])
+        })
+        return array.join("")
+    } else {
+        const keys = Object.keys(xmlObject)
+        // console.log("Keys: ", keys)
+        return keys.map((key, index, array) => {
+            if (key == "component") {
+                return produceString(xmlObject[key].slice(3), references)
+            } else if (key == "header") {
+                return produceString(xmlObject[key], references)
+            } else if (key =="contentMeta") {
+                return produceString(xmlObject[key], references)
+            } else if (key =="titleGroup") {
+                return produceString(xmlObject[key], references)
+            } else if (key =="title") {
+                return `# ${produceString(xmlObject[key], references)}`
+            } else if (key =="body") {
+                return produceString(xmlObject[key], references)
+            } else if (key == "?xmltex") {
+                return `${produceString(xmlObject[key], references)} `
+            } else if (key == "#text") {
+                return xmlObject[key]
+            } else if (key == "section") {
+                return `\n${produceString(xmlObject[key], references, level+1)}`
+            } else if (key == "i") {
+                return ` *${produceString(xmlObject[key], references)}*${appendOptionalSpace(nextObject)}`
+            } else if (key == "p") {
+                return `\n${produceString(xmlObject[key], references)}\n`
+            } else if (key == ":@") {
+                return `${produceString(xmlObject[key], references)}`
+            } else if (key == "exlink") {
+                const chapterNumber = xmlObject[":@"]["@_href"].split(":").at(-1).slice(-2)
+                return ` [${Number(chapterNumber)}](exlink://${xmlObject[":@"]["@_href"]})${appendOptionalSpace(nextObject)}`
+            } else if (key == "url") {
+                return ` [${produceString(xmlObject["url"], references)}][${xmlObject[":@"]["@_href"]}] `
+            } else if (key == "link") {
+                return ` [${xmlObject[":@"]["@_href"]}](link://${xmlObject[":@"]["@_href"]})${appendOptionalSpace(nextObject)}`
+            } else if (key == "note") {
+                const footnote = `\n[^${xmlObject[":@"]['@_xml:id']}]: ${produceString(xmlObject[key], references)}\n`
+                references.push(footnote)
+                return `[^${xmlObject[":@"]['@_xml:id']}]${appendOptionalSpace(nextObject)}`
+            } else if (key == "figure") {
+                const image = xmlObject[key].find(o => o.mediaResource)
+                const caption = xmlObject[key].find(o => o.caption)
+
+                if (caption) {
+                    return `\n${produceString(image, references)}\n<figure>\n${produceString(caption)}\n</figure>\n`
+                } else {
+                    return `\n${produceString(image, references)}\n`
+                }
+            } else if (key == "mediaResource") {
+                return `![](${xmlObject[":@"]["@_href"]})`
+            } else if (key == "caption") {
+                return `<figcaption>${produceString(xmlObject[key], references)}</figcaption>`
+            } 
+            return ""
+        }).join("")
+    }
+}
+
+const btcRootDir = "/home/sigidli/Documents/business/exonumia/bitcoin-standard/9781119473862"
+const excludeRootContent = [
+    "pdf",
+    "Manifest",
+    "metadata",
+    "cover_US",
+    "control",
+    "summary.xls",
+    "protocol.html",
+    // "fmatter",
+    // "bmatter"
+]
+const rootDirectories = fs.readdirSync(btcRootDir).filter(d => !excludeRootContent.includes(d)) // .filter(d => d == "ch03")
+
+rootDirectories.forEach(directory => {
+    console.log(directory)
+    const contentDirectories = fs.readdirSync(path.join(btcRootDir, directory))
+
+    // TODO: image export
+    const textFiles = fs.readdirSync(path.join(btcRootDir, `${directory}/text_s`))
+
+    textFiles.forEach(file => {
+        const fileContent = fs.readFileSync(path.join(btcRootDir, `${directory}/text_s/${file}`))
+        const fileStringContent = fileContent.toString().split("\n").slice(6).join("\n")
+
+        const parser = new XMLParser({
+            ignoreAttributes: false,
+            preserveOrder: true,
+        });
+            
+        let xmlObject = parser.parse(fileStringContent)
+        const references = []
+        var markdownContent = produceString(xmlObject, references)
+        markdownContent += references.join("")
+
+        const resultDIR = "result"
+        if (!fs.existsSync(resultDIR)) {
+            fs.mkdirSync(resultDIR)
+        }
+        
+        fs.writeFileSync(
+            path.join(resultDIR, `${file}.md`),
+            markdownContent
+        )
+    })
+});
+

server/router/projects/index.js

@@ -1569,7 +1569,7 @@ module.exports = function (options) {
                     for (const sheetName in workbook.Sheets) {
                         const sheet = workbook.Sheets[sheetName];
                         
-                        const artifactVersionId = sheet["D4"].v
+                        if (sheet["E4"]) {
                             const translationArtifactVersionId = sheet["E4"].v
 
                             var lastTranslationChunkId = null
@@ -1592,6 +1592,10 @@ module.exports = function (options) {
                                                         association: db.TranslationChunk.Translation
                                                     }
                                                 ]
+                                            }).catch(error => {
+                                                console.error("Couldn't process: ",translationChunkCell);
+                                                console.error(`Error ${lastTranslationChunkId}: `, error);
+                                                return null;
                                             })
 
                                             if (translationChunk) {
@@ -1628,6 +1632,10 @@ module.exports = function (options) {
                             // Chunks...
 
                             console.log("Done processing all...")
+                        } else {
+                            console.log("Couldn't process sheet: ", sheetName)
+                        }
+                        
                     }
 
                     response.redirect(`/projects/${project.id}`)

server/router/translate/index.js

@@ -320,7 +320,7 @@ module.exports = function (options) {
             }).then(translationChapter => {
                 if (translationChapter) {
                     response.contentType = "text/plain"
-                    response.send(markdownProducer.produceMarkdownString(tokens))
+                    response.send(markdownProducer.produceMarkdownString(translationChapter))
                 } else {
                     next()
                 }