natty
/
calckey
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support new zip file by mastodon

This commit is contained in:
CGsama 2023-07-16 06:24:37 -04:00
parent ffb7fb5309
commit 4bb6d7105a
4 changed files with 196 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/backend/package.json
View File

@ -59,6 +59,7 @@
"color-convert": "2.0.1",
"content-disposition": "0.5.4",
"date-fns": "2.30.0",
"decompress": "^4.2.1",
"deep-email-validator": "0.1.21",
"escape-regexp": "0.0.1",
"feed": "4.2.2",

42
packages/backend/src/misc/process-masto-notes.ts
View File

@ -6,10 +6,13 @@ import { addFile } from "@/services/drive/add-file.js";
import { Users } from "@/models/index.js";
import * as tar from "tar-stream";
import gunzip from "gunzip-maybe";
import decompress from "decompress";
import * as Path from "node:path";
const logger = new Logger("process-masto-notes");
export async function processMastoNotes(
fn: string,
url: string,
uid: string,
): Promise<any> {
@ -23,26 +26,39 @@ export async function processMastoNotes(
try {
// write content at URL to temp file
await downloadUrl(url, path);
return await processMastoFile(path, unzipPath, uid);
return await processMastoFile(fn, path, unzipPath, uid);
} finally {
cleanup();
unzipCleanup();
//unzipCleanup();
}
}
function processMastoFile(fn: string, dir: string, uid: string) {
function processMastoFile(fn: string, path: string, dir: string, uid: string) {
return new Promise(async (resolve, reject) => {
const user = await Users.findOneBy({ id: uid });
try {
logger.info(`Start unzip ${fn}`);
await unzipTarGz(fn, dir);
logger.info(`Start unzip ${path}`);
fn.endsWith("tar.gz")
? await unzipTarGz(path, dir)
: await unzipZip(path, dir);
logger.info(`Unzip to ${dir}`);
const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
for (const note of outbox.orderedItems) {
for (const attachment of note.object.attachment) {
const url = attachment.url.replace("..", "");
const url = attachment.url.replaceAll("..", "");
if (url.indexOf('\0') !== -1) {
logger.error(`Found Poison Null Bytes Attack: ${url}`);
reject();
return;
}
try {
const fpath = `${dir}${url}`;
const fpath = Path.resolve(`${dir}${url}`);
if (!fpath.startsWith(dir)) {
logger.error(`Found Path Attack: ${url}`);
reject();
return;
}
logger.info(fpath);
const driveFile = await addFile({ user: user, path: fpath });
attachment.driveFile = driveFile;
} catch (e) {
@ -65,6 +81,18 @@ function createFileDir(fn: string) {
}
}
function unzipZip(fn: string, dir: string) {
return new Promise(async (resolve, reject) => {
try {
decompress(fn, dir).then((files: any) => {
resolve(files);
});
} catch (e) {
reject();
}
});
}
function unzipTarGz(fn: string, dir: string) {
return new Promise(async (resolve, reject) => {
const onErr = (err: any) => {

8
packages/backend/src/queue/processors/db/import-posts.ts
View File

@ -31,10 +31,14 @@ export async function importPosts(
return;
}
if (file.name.endsWith("tar.gz")) {
if (file.name.endsWith("tar.gz") || file.name.endsWith("zip")) {
try {
logger.info("Reading Mastodon archive");
const outbox = await processMastoNotes(file.url, job.data.user.id);
const outbox = await processMastoNotes(
file.name,
file.url,
job.data.user.id,
);
for (const post of outbox.orderedItems) {
createImportMastoPostJob(job.data.user, post, job.data.signatureCheck);
}

161
pnpm-lock.yaml
View File

@ -183,6 +183,9 @@ importers:
date-fns:
specifier: 2.30.0
version: 2.30.0
decompress:
specifier: ^4.2.1
version: 4.2.1
deep-email-validator:
specifier: 0.1.21
version: 0.1.21
@ -5402,6 +5405,13 @@ packages:
engines: {node: '>=0.8'}
dev: true
/bl@1.2.3:
resolution: {integrity: sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==}
dependencies:
readable-stream: 2.3.8
safe-buffer: 5.2.1
dev: false
/bl@4.1.0:
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
dependencies:
@ -5527,6 +5537,17 @@ packages:
node-int64: 0.4.0
dev: true
/buffer-alloc-unsafe@1.1.0:
resolution: {integrity: sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==}
dev: false
/buffer-alloc@1.2.0:
resolution: {integrity: sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==}
dependencies:
buffer-alloc-unsafe: 1.1.0
buffer-fill: 1.0.0
dev: false
/buffer-crc32@0.2.13:
resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==}
@ -5539,6 +5560,10 @@ packages:
engines: {node: '>=0.4'}
dev: true
/buffer-fill@1.0.0:
resolution: {integrity: sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==}
dev: false
/buffer-from@1.1.2:
resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
@ -6930,6 +6955,59 @@ packages:
dependencies:
mimic-response: 3.1.0
/decompress-tar@4.1.1:
resolution: {integrity: sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==}
engines: {node: '>=4'}
dependencies:
file-type: 5.2.0
is-stream: 1.1.0
tar-stream: 1.6.2
dev: false
/decompress-tarbz2@4.1.1:
resolution: {integrity: sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==}
engines: {node: '>=4'}
dependencies:
decompress-tar: 4.1.1
file-type: 6.2.0
is-stream: 1.1.0
seek-bzip: 1.0.6
unbzip2-stream: 1.4.3
dev: false
/decompress-targz@4.1.1:
resolution: {integrity: sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==}
engines: {node: '>=4'}
dependencies:
decompress-tar: 4.1.1
file-type: 5.2.0
is-stream: 1.1.0
dev: false
/decompress-unzip@4.0.1:
resolution: {integrity: sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==}
engines: {node: '>=4'}
dependencies:
file-type: 3.9.0
get-stream: 2.3.1
pify: 2.3.0
yauzl: 2.10.0
dev: false
/decompress@4.2.1:
resolution: {integrity: sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==}
engines: {node: '>=4'}
dependencies:
decompress-tar: 4.1.1
decompress-tarbz2: 4.1.1
decompress-targz: 4.1.1
decompress-unzip: 4.0.1
graceful-fs: 4.2.11
make-dir: 1.3.0
pify: 2.3.0
strip-dirs: 2.1.0
dev: false
/dedent@0.7.0:
resolution: {integrity: sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==}
dev: true
@ -8161,7 +8239,6 @@ packages:
resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==}
dependencies:
pend: 1.2.0
dev: true
/feed@4.2.2:
resolution: {integrity: sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==}
@ -8211,6 +8288,21 @@ packages:
strtok3: 7.0.0
token-types: 5.0.1
/file-type@3.9.0:
resolution: {integrity: sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==}
engines: {node: '>=0.10.0'}
dev: false
/file-type@5.2.0:
resolution: {integrity: sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==}
engines: {node: '>=4'}
dev: false
/file-type@6.2.0:
resolution: {integrity: sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==}
engines: {node: '>=4'}
dev: false
/filelist@1.0.4:
resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==}
dependencies:
@ -8679,6 +8771,14 @@ packages:
engines: {node: '>=8'}
dev: false
/get-stream@2.3.1:
resolution: {integrity: sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==}
engines: {node: '>=0.10.0'}
dependencies:
object-assign: 4.1.1
pinkie-promise: 2.0.1
dev: false
/get-stream@3.0.0:
resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==}
engines: {node: '>=4'}
@ -9841,6 +9941,10 @@ packages:
resolution: {integrity: sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==}
dev: false
/is-natural-number@4.0.1:
resolution: {integrity: sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==}
dev: false
/is-negated-glob@1.0.0:
resolution: {integrity: sha512-czXVVn/QEmgvej1f50BZ648vUI+em0xqMq2Sn+QncCLN4zj1UAxlT+kw/6ggQTOaZPd1HqKQGEqbpQVtJucWug==}
engines: {node: '>=0.10.0'}
@ -9942,7 +10046,6 @@ packages:
/is-stream@1.1.0:
resolution: {integrity: sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==}
engines: {node: '>=0.10.0'}
dev: true
/is-stream@2.0.1:
resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
@ -12023,6 +12126,13 @@ packages:
resolution: {integrity: sha512-3WjL8+ZDouZwKlyJBMp/4LeziLFXgleOdsYu87piGcMLqhBzCsy2QFdbtAwv757TFC/rtqd738fgJw1tFQCSgA==}
dev: false
/make-dir@1.3.0:
resolution: {integrity: sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==}
engines: {node: '>=4'}
dependencies:
pify: 3.0.0
dev: false
/make-dir@3.1.0:
resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
engines: {node: '>=8'}
@ -13351,7 +13461,6 @@ packages:
/pend@1.2.0:
resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}
dev: true
/performance-now@2.1.0:
resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==}
@ -13435,7 +13544,11 @@ packages:
/pify@2.3.0:
resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==}
engines: {node: '>=0.10.0'}
dev: true
/pify@3.0.0:
resolution: {integrity: sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==}
engines: {node: '>=4'}
dev: false
/pify@4.0.1:
resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==}
@ -13446,12 +13559,10 @@ packages:
engines: {node: '>=0.10.0'}
dependencies:
pinkie: 2.0.4
dev: true
/pinkie@2.0.4:
resolution: {integrity: sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==}
engines: {node: '>=0.10.0'}
dev: true
/pirates@4.0.6:
resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==}
@ -14818,6 +14929,13 @@ packages:
/seedrandom@3.0.5:
resolution: {integrity: sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==}
/seek-bzip@1.0.6:
resolution: {integrity: sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==}
hasBin: true
dependencies:
commander: 2.20.3
dev: false
/semver-greatest-satisfied-range@1.1.0:
resolution: {integrity: sha512-Ny/iyOzSSa8M5ML46IAx3iXc6tfOsYU2R4AXi2UpHk60Zrgyq6eqPj/xiOfS0rRl/iiQ/rdJkVjw/5cdUyCntQ==}
engines: {node: '>= 0.10'}
@ -15450,6 +15568,12 @@ packages:
engines: {node: '>=8'}
dev: true
/strip-dirs@2.1.0:
resolution: {integrity: sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==}
dependencies:
is-natural-number: 4.0.1
dev: false
/strip-eof@1.0.0:
resolution: {integrity: sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==}
engines: {node: '>=0.10.0'}
@ -15643,6 +15767,19 @@ packages:
tar-stream: 2.2.0
dev: false
/tar-stream@1.6.2:
resolution: {integrity: sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==}
engines: {node: '>= 0.8.0'}
dependencies:
bl: 1.2.3
buffer-alloc: 1.2.0
end-of-stream: 1.4.4
fs-constants: 1.0.0
readable-stream: 2.3.8
to-buffer: 1.1.1
xtend: 4.0.2
dev: false
/tar-stream@2.2.0:
resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
engines: {node: '>=6'}
@ -15875,6 +16012,10 @@ packages:
is-negated-glob: 1.0.0
dev: true
/to-buffer@1.1.1:
resolution: {integrity: sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==}
dev: false
/to-fast-properties@2.0.0:
resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==}
engines: {node: '>=4'}
@ -16413,6 +16554,13 @@ packages:
which-boxed-primitive: 1.0.2
dev: true
/unbzip2-stream@1.4.3:
resolution: {integrity: sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==}
dependencies:
buffer: 5.7.1
through: 2.3.8
dev: false
/unc-path-regex@0.1.2:
resolution: {integrity: sha512-eXL4nmJT7oCpkZsHZUOJo8hcX3GbsiDOa0Qu9F646fi8dT3XuSVopVqAcEiVzSKKH7UoDti23wNX3qGFxcW5Qg==}
engines: {node: '>=0.10.0'}
@ -17424,7 +17572,6 @@ packages:
dependencies:
buffer-crc32: 0.2.13
fd-slicer: 1.1.0
dev: true
/ylru@1.3.2:
resolution: {integrity: sha512-RXRJzMiK6U2ye0BlGGZnmpwJDPgakn6aNQ0A7gHRbD4I0uvK4TW6UqkK1V0pp9jskjJBAXd3dRrbzWkqJ+6cxA==}