author: ${comment.author}, -subreddit: ${comment.subreddit}, -karma: ${comment.score}, created at ${dateTimeStr}
` - + unescape(comment.body_html); - - let parentNoteId = await getDateNoteIdForReddit(dateTimeStr, rootNoteId); - - await sql.doInTransaction(async () => { - commentNoteId = await createNote(parentNoteId, comment.link_title, noteText); - - log.info("Reddit: Imported comment to note " + commentNoteId); - importedComments++; - - await attributes.createAttribute(commentNoteId, "reddit_kind", child.kind); - await attributes.createAttribute(commentNoteId, "reddit_id", redditId(child.kind, comment.id)); - await attributes.createAttribute(commentNoteId, "reddit_created_utc", comment.created_utc); - }); - } - - // if there have been no imported comments on this page, there shouldn't be any to import - // on the next page since those are older - if (listing.data.after && importedComments > 0) { - importedComments += await importComments(rootNoteId, accountName, listing.data.after); - } - - return importedComments; -} - -let redditAccounts = []; - -async function runImport() { - const rootNoteId = await date_notes.getRootCalendarNoteId(); - - // technically mutex shouldn't be necessary but we want to avoid doing potentially expensive import - // concurrently with sync - await sync_mutex.doExclusively(async () => { - let importedComments = 0; - - for (const account of redditAccounts) { - importedComments += await importComments(rootNoteId, account); - } - - log.info(`Reddit: Imported ${importedComments} comments.`); - }); -} - -sql.dbReady.then(async () => { - if (!config['Reddit'] || config['Reddit']['enabled'] !== true) { - return; - } - - const redditAccountsStr = config['Reddit']['accounts']; - - if (!redditAccountsStr) { - log.info("Reddit: No reddit accounts defined in option 'reddit_accounts'"); - } - - redditAccounts = redditAccountsStr.split(",").map(s => s.trim()); - - const pollingIntervalInSeconds = config['Reddit']['pollingIntervalInSeconds'] || (4 * 3600); - - setInterval(runImport, pollingIntervalInSeconds * 1000); - setTimeout(runImport, 10000); // 10 seconds after startup - intentionally after initial sync -}); diff --git a/src/routes/api/export.js b/src/routes/api/export.js index 344a8da76..47b241361 100644 --- a/src/routes/api/export.js +++ b/src/routes/api/export.js @@ -35,17 +35,20 @@ async function exportNote(noteTreeId, directory, pack) { return; } - const content = note.type === 'text' ? html.prettyPrint(note.content, {indent_size: 2}) : note.content; - - const childFileName = directory + sanitize(note.title); - - console.log(childFileName); - - pack.entry({ name: childFileName + ".dat", size: content.length }, content); - const metadata = await getMetadata(note); - pack.entry({ name: childFileName + ".meta", size: metadata.length }, metadata); + if ('exclude_from_export' in metadata.attributes) { + return; + } + + const metadataJson = JSON.stringify(metadata, null, '\t'); + const childFileName = directory + sanitize(note.title); + + pack.entry({ name: childFileName + ".meta", size: metadataJson.length }, metadataJson); + + const content = note.type === 'text' ? html.prettyPrint(note.content, {indent_size: 2}) : note.content; + + pack.entry({ name: childFileName + ".dat", size: content.length }, content); const children = await sql.getRows("SELECT * FROM note_tree WHERE parentNoteId = ? AND isDeleted = 0", [note.noteId]); @@ -59,14 +62,12 @@ async function exportNote(noteTreeId, directory, pack) { } async function getMetadata(note) { - const meta = { + return { title: note.title, type: note.type, mime: note.mime, attributes: await attributes.getNoteAttributeMap(note.noteId) }; - - return JSON.stringify(meta, null, '\t') } module.exports = router; \ No newline at end of file diff --git a/src/scripts/Reddit Importer.tar b/src/scripts/Reddit Importer.tar new file mode 100644 index 000000000..b8145e982 Binary files /dev/null and b/src/scripts/Reddit Importer.tar differ diff --git a/src/services/attributes.js b/src/services/attributes.js index f1f752c0b..c597c766e 100644 --- a/src/services/attributes.js +++ b/src/services/attributes.js @@ -9,7 +9,8 @@ const BUILTIN_ATTRIBUTES = [ 'run_on_startup', 'disable_versioning', 'calendar_root', - 'hide_in_autocomplete' + 'hide_in_autocomplete', + 'exclude_from_export' ]; async function getNoteAttributeMap(noteId) { diff --git a/src/services/notes.js b/src/services/notes.js index 38ecf9b37..3f38f1039 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -84,6 +84,9 @@ async function createNewNote(parentNoteId, noteOpts, dataKey, sourceId) { } async function createNote(parentNoteId, title, content = "", extraOptions = {}) { + if (!parentNoteId) throw new Error("Empty parentNoteId"); + if (!title) throw new Error("Empty title"); + const note = { title: title, content: extraOptions.json ? JSON.stringify(content, null, '\t') : content, diff --git a/src/services/script_context.js b/src/services/script_context.js index 921504ba7..8a5ddcc07 100644 --- a/src/services/script_context.js +++ b/src/services/script_context.js @@ -2,15 +2,24 @@ const log = require('./log'); const protected_session = require('./protected_session'); const notes = require('./notes'); const sql = require('./sql'); +const utils = require('./utils'); const attributes = require('./attributes'); const date_notes = require('./date_notes'); const config = require('./config'); const Repository = require('./repository'); +const axios = require('axios'); function ScriptContext(dataKey) { dataKey = protected_session.getDataKey(dataKey); const repository = new Repository(dataKey); + this.axios = axios; + + this.utils = { + unescapeHtml: utils.unescapeHtml, + isoDateTimeStr: utils.dateStr + }; + this.getInstanceName = () => config.General ? config.General.instanceName : null; this.getNoteById = async function(noteId) { @@ -30,7 +39,7 @@ function ScriptContext(dataKey) { this.createNote = async function(parentNoteId, title, content = "", extraOptions = {}) { extraOptions.dataKey = dataKey; - notes.createNote(parentNoteId, title, content, extraOptions); + return await notes.createNote(parentNoteId, title, content, extraOptions); }; this.createAttribute = attributes.createAttribute; diff --git a/src/services/utils.js b/src/services/utils.js index 5596ca933..a6c4e0645 100644 --- a/src/services/utils.js +++ b/src/services/utils.js @@ -2,6 +2,7 @@ const crypto = require('crypto'); const randtoken = require('rand-token').generator({source: 'crypto'}); +const unescape = require('unescape'); function newNoteId() { return randomString(12); @@ -129,6 +130,10 @@ async function stopWatch(what, func) { return ret; } +function unescapeHtml(str) { + return unescape(str); +} + module.exports = { randomSecureToken, randomString, @@ -153,5 +158,6 @@ module.exports = { getDateTimeForFile, sanitizeSql, assertArguments, - stopWatch + stopWatch, + unescapeHtml }; \ No newline at end of file