natty
/
calckey
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'Add raw zip emoji imports' (#10246) from PrivateGER/calckey:fix/no-metadata-emoji-import into develop 

Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10246
This commit is contained in:
Kainoa Kanter 2023-06-01 20:43:57 +00:00
parent 35e72bfca7 7eeac4810a
commit e3b337e115
1 changed files with 89 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
packages/backend/src/queue/processors/db/import-custom-emojis.ts
View File

@ -11,6 +11,7 @@ import { addFile } from "@/services/drive/add-file.js";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import { db } from "@/db/postgre.js"; import { db } from "@/db/postgre.js";
import probeImageSize from "probe-image-size"; import probeImageSize from "probe-image-size";
import * as path from "path";
const logger = queueLogger.createSubLogger("import-custom-emojis"); const logger = queueLogger.createSubLogger("import-custom-emojis");
@ -29,11 +30,11 @@ export async function importCustomEmojis(
return; return;
} }
const [path, cleanup] = await createTempDir(); const [tempPath, cleanup] = await createTempDir();
logger.info(`Temp dir is ${path}`); logger.info(`Temp dir is ${tempPath}`);
const destPath = `${path}/emojis.zip`; const destPath = `${tempPath}/emojis.zip`;
try { try {
fs.writeFileSync(destPath, "", "binary"); fs.writeFileSync(destPath, "", "binary");
@ -46,44 +47,96 @@ export async function importCustomEmojis(
throw e; throw e;
} }
const outputPath = `${path}/emojis`; const outputPath = `${tempPath}/emojis`;
const unzipStream = fs.createReadStream(destPath); const unzipStream = fs.createReadStream(destPath);
const zip = new AdmZip(destPath); const zip = new AdmZip(destPath);
zip.extractAllToAsync(outputPath, true, false, async (error) => { zip.extractAllToAsync(outputPath, true, false, async (error) => {
if (error) throw error; if (error) throw error;
const metaRaw = fs.readFileSync(`${outputPath}/meta.json`, "utf-8");
const meta = JSON.parse(metaRaw);
for (const record of meta.emojis) { if (fs.existsSync(`${outputPath}/meta.json`)) {
if (!record.downloaded) continue; logger.info("starting emoji import with metadata");
const emojiInfo = record.emoji; const metaRaw = fs.readFileSync(`${outputPath}/meta.json`, "utf-8");
const emojiPath = `${outputPath}/${record.fileName}`; const meta = JSON.parse(metaRaw);
await Emojis.delete({
name: emojiInfo.name, for (const record of meta.emojis) {
}); if (!record.downloaded) continue;
const driveFile = await addFile({ const emojiInfo = record.emoji;
user: null, const emojiPath = `${outputPath}/${record.fileName}`;
path: emojiPath, await Emojis.delete({
name: record.fileName, name: emojiInfo.name,
force: true, });
}); const driveFile = await addFile({
const file = fs.createReadStream(emojiPath); user: null,
const size = await probeImageSize(file); path: emojiPath,
file.destroy(); name: record.fileName,
await Emojis.insert({ force: true,
id: genId(), });
updatedAt: new Date(), const file = fs.createReadStream(emojiPath);
name: emojiInfo.name, const size = await probeImageSize(file);
category: emojiInfo.category, file.destroy();
host: null, await Emojis.insert({
aliases: emojiInfo.aliases, id: genId(),
originalUrl: driveFile.url, updatedAt: new Date(),
publicUrl: driveFile.webpublicUrl ?? driveFile.url, name: emojiInfo.name,
type: driveFile.webpublicType ?? driveFile.type, category: emojiInfo.category,
license: emojiInfo.license, host: null,
width: size.width || null, aliases: emojiInfo.aliases,
height: size.height || null, originalUrl: driveFile.url,
}).then((x) => Emojis.findOneByOrFail(x.identifiers[0])); publicUrl: driveFile.webpublicUrl ?? driveFile.url,
type: driveFile.webpublicType ?? driveFile.type,
license: emojiInfo.license,
width: size.width || null,
height: size.height || null,
}).then((x) => Emojis.findOneByOrFail(x.identifiers[0]));
}
} else {
logger.info("starting emoji import without metadata");
// Since we lack metadata, we import into a randomized category name instead
let categoryName = genId();
let containedEmojis = fs.readdirSync(outputPath);
// Filter out accidental JSON files
containedEmojis = containedEmojis.filter(
(emoji) => !emoji.match(/\.(json)$/i),
);
for (const emojiFilename of containedEmojis) {
// strip extension and get filename to use as name
const name = path.basename(emojiFilename, path.extname(emojiFilename));
const emojiPath = `${outputPath}/${emojiFilename}`;
logger.info(`importing ${name}`);
await Emojis.delete({
name: name,
});
const driveFile = await addFile({
user: null,
path: emojiPath,
name: path.basename(emojiFilename),
force: true,
});
const file = fs.createReadStream(emojiPath);
const size = await probeImageSize(file);
file.destroy();
logger.info(`emoji size: ${size.width}x${size.height}`);
await Emojis.insert({
id: genId(),
updatedAt: new Date(),
name: name,
category: categoryName,
host: null,
aliases: [],
originalUrl: driveFile.url,
publicUrl: driveFile.webpublicUrl ?? driveFile.url,
type: driveFile.webpublicType ?? driveFile.type,
license: null,
width: size.width || null,
height: size.height || null,
}).then((x) => Emojis.findOneByOrFail(x.identifiers[0]));
}
} }
await db.queryResultCache!.remove(["meta_emojis"]); await db.queryResultCache!.remove(["meta_emojis"]);