starlight/reddit-lemmy-importer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deal with posts containing "$$$" and handle submissions without a name field too

Browse Source
This commit is contained in:
starlight 2025-01-22 11:07:04 +13:00
parent c76e3e7cbb
commit 3925c2f8ed
2 changed files with 21 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/parser.js
View File

@ -68,13 +68,18 @@ async function processPostsAndComments(postsFile, commentsFile) {
crlfDelay: Infinity
});
var dbgpost = 0;
//var dbgpost = 0;
for await (const line of postsStream) {
if (line.trim()) {
const post = filterJsonKeys(JSON.parse(line), submissionKeysAllowed);
// i think this is only a problem for the comments (see below) but i did it here too as a safety measure
var post = JSON.parse(line)
if(!post.name){
post.name = `t3_${post.id}`;
}
post = filterJsonKeys(post, submissionKeysAllowed);
context.processItem(post);
//dbgpost==467?console.log(dbgpost + line):null;
dbgpost++;
//dbgpost++;
}
}
@ -86,7 +91,14 @@ async function processPostsAndComments(postsFile, commentsFile) {
for await (const line of commentsStream) {
if (line.trim()) {
const comment = filterJsonKeys(JSON.parse(line), commentKeysAllowed);
// dont filter yet so that we can have the id key
var comment = JSON.parse(line)
// if its a comment with no "name" then make a "name" field
if(!comment.name){
comment.name = `t1_${comment.id}`;
}
comment = filterJsonKeys(comment, commentKeysAllowed);
context.processItem(comment);
}
}

6
src/sql.js
View File

@ -17,8 +17,10 @@ function lit(str) {
} */
// decodeHTML then replace all instances of ' with ''
// then escape $ as \$, since saying "$$$" (like money) will close the "DO $$"" statement :(
function lit(str) {
return typeof str === 'string' ? decodeHTML(str).replace(/'/g, "''") : 'null'
return typeof str === 'string' ? decodeHTML(str).replace(/'/g, "''").replace(/\$/g, "\\$") : 'null'
}
// Decode HTML entities (e.g., '>' -> '>')
@ -52,7 +54,7 @@ function mkTitle(post) {
}
// wrap the url in singlequotes HERE and not in the query like '${lit(mkUrl(post))}'
// this is because the null type in postgres will be turned into a string which will break lemmy until you remove the row or set it to null manually
// this is because the null type in postgres will be turned into a string which will break lemmy until you remove the row or set it to null manually
function mkUrl(post) {
return post.is_gallery ? `'${getSubmissionImages(post)[0]}'` : post.is_self ? 'null' : `'${post.url}'`
}