Lint + formatting
This commit is contained in:
parent
43813d12b5
commit
274bfcd898
|
@ -41,7 +41,7 @@ export default function () {
|
||||||
r: round(Math.max(0, fsStats.rIO_sec ?? 0)),
|
r: round(Math.max(0, fsStats.rIO_sec ?? 0)),
|
||||||
w: round(Math.max(0, fsStats.wIO_sec ?? 0)),
|
w: round(Math.max(0, fsStats.wIO_sec ?? 0)),
|
||||||
},
|
},
|
||||||
meilisearch: meilisearchStats
|
meilisearch: meilisearchStats,
|
||||||
};
|
};
|
||||||
ev.emit("serverStats", stats);
|
ev.emit("serverStats", stats);
|
||||||
log.unshift(stats);
|
log.unshift(stats);
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import {Health, MeiliSearch, Stats} from 'meilisearch';
|
import {Health, MeiliSearch, Stats} from "meilisearch";
|
||||||
import {dbLogger} from "./logger.js";
|
import {dbLogger} from "./logger.js";
|
||||||
|
|
||||||
import config from "@/config/index.js";
|
import config from "@/config/index.js";
|
||||||
|
@ -7,13 +7,15 @@ import * as url from "url";
|
||||||
import {User} from "@/models/entities/user.js";
|
import {User} from "@/models/entities/user.js";
|
||||||
import {Users} from "@/models/index.js";
|
import {Users} from "@/models/index.js";
|
||||||
|
|
||||||
|
|
||||||
const logger = dbLogger.createSubLogger("meilisearch", "gray", false);
|
const logger = dbLogger.createSubLogger("meilisearch", "gray", false);
|
||||||
|
|
||||||
logger.info("Connecting to MeiliSearch");
|
logger.info("Connecting to MeiliSearch");
|
||||||
|
|
||||||
const hasConfig =
|
const hasConfig =
|
||||||
config.meilisearch && (config.meilisearch.host || config.meilisearch.port || config.meilisearch.apiKey);
|
config.meilisearch &&
|
||||||
|
(config.meilisearch.host ||
|
||||||
|
config.meilisearch.port ||
|
||||||
|
config.meilisearch.apiKey);
|
||||||
|
|
||||||
const host = hasConfig ? config.meilisearch.host ?? "localhost" : "";
|
const host = hasConfig ? config.meilisearch.host ?? "localhost" : "";
|
||||||
const port = hasConfig ? config.meilisearch.port ?? 7700 : 0;
|
const port = hasConfig ? config.meilisearch.port ?? 7700 : 0;
|
||||||
|
@ -22,13 +24,28 @@ const auth = hasConfig ? config.meilisearch.apiKey ?? "" : "";
|
||||||
const client: MeiliSearch = new MeiliSearch({
|
const client: MeiliSearch = new MeiliSearch({
|
||||||
host: `http://${host}:${port}`,
|
host: `http://${host}:${port}`,
|
||||||
apiKey: auth,
|
apiKey: auth,
|
||||||
})
|
});
|
||||||
|
|
||||||
const posts = client.index('posts');
|
const posts = client.index("posts");
|
||||||
|
|
||||||
posts.updateSearchableAttributes(['text']).catch((e) => logger.error(`Setting searchable attr failed, searches won't work: ${e}`));
|
posts
|
||||||
|
.updateSearchableAttributes(["text"])
|
||||||
|
.catch((e) =>
|
||||||
|
logger.error(`Setting searchable attr failed, searches won't work: ${e}`),
|
||||||
|
);
|
||||||
|
|
||||||
posts.updateFilterableAttributes(["userName", "userHost", "mediaAttachment", "createdAt"]).catch((e) => logger.error(`Setting filterable attr failed, advanced searches won't work: ${e}`));
|
posts
|
||||||
|
.updateFilterableAttributes([
|
||||||
|
"userName",
|
||||||
|
"userHost",
|
||||||
|
"mediaAttachment",
|
||||||
|
"createdAt",
|
||||||
|
])
|
||||||
|
.catch((e) =>
|
||||||
|
logger.error(
|
||||||
|
`Setting filterable attr failed, advanced searches won't work: ${e}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
logger.info("Connected to MeiliSearch");
|
logger.info("Connected to MeiliSearch");
|
||||||
|
|
||||||
|
@ -40,128 +57,129 @@ export type MeilisearchNote = {
|
||||||
userName: string;
|
userName: string;
|
||||||
channelId: string;
|
channelId: string;
|
||||||
mediaAttachment: string;
|
mediaAttachment: string;
|
||||||
createdAt: number
|
createdAt: number;
|
||||||
}
|
};
|
||||||
|
|
||||||
export default hasConfig ? {
|
export default hasConfig
|
||||||
search: (query : string, limit : number, offset : number) => {
|
? {
|
||||||
|
search: (query: string, limit: number, offset: number) => {
|
||||||
|
/// Advanced search syntax
|
||||||
|
/// from:user => filter by user + optional domain
|
||||||
|
/// has:image/video/audio/text/file => filter by attachment types
|
||||||
|
/// domain:domain.com => filter by domain
|
||||||
|
/// before:Date => show posts made before Date
|
||||||
|
/// after: Date => show posts made after Date
|
||||||
|
|
||||||
/// Advanced search syntax
|
let constructedFilters: string[] = [];
|
||||||
/// from:user => filter by user + optional domain
|
|
||||||
/// has:image/video/audio/text/file => filter by attachment types
|
|
||||||
/// domain:domain.com => filter by domain
|
|
||||||
/// before:Date => show posts made before Date
|
|
||||||
/// after: Date => show posts made after Date
|
|
||||||
|
|
||||||
|
let splitSearch = query.split(" ");
|
||||||
|
|
||||||
let constructedFilters: string[] = [];
|
// Detect search operators and remove them from the actual query
|
||||||
|
splitSearch = splitSearch.filter((term) => {
|
||||||
let splitSearch = query.split(" ");
|
if (term.startsWith("has:")) {
|
||||||
|
let fileType = term.slice(4);
|
||||||
// Detect search operators and remove them from the actual query
|
constructedFilters.push(`mediaAttachment = "${fileType}"`);
|
||||||
splitSearch = splitSearch.filter(term => {
|
return false;
|
||||||
if (term.startsWith("has:")) {
|
} else if (term.startsWith("from:")) {
|
||||||
let fileType = term.slice(4);
|
let user = term.slice(5);
|
||||||
constructedFilters.push(`mediaAttachment = "${fileType}"`)
|
constructedFilters.push(`userName = ${user}`);
|
||||||
return false;
|
return false;
|
||||||
} else if (term.startsWith("from:")) {
|
} else if (term.startsWith("domain:")) {
|
||||||
let user = term.slice(5);
|
let domain = term.slice(7);
|
||||||
constructedFilters.push(`userName = ${user}`)
|
constructedFilters.push(`userHost = ${domain}`);
|
||||||
return false;
|
return false;
|
||||||
} else if (term.startsWith("domain:")) {
|
} else if (term.startsWith("after:")) {
|
||||||
let domain = term.slice(7);
|
let timestamp = term.slice(6);
|
||||||
constructedFilters.push(`userHost = ${domain}`)
|
// Try to parse the timestamp as JavaScript Date
|
||||||
return false;
|
let date = Date.parse(timestamp);
|
||||||
} else if (term.startsWith("after:")) {
|
if (isNaN(date)) return false;
|
||||||
let timestamp = term.slice(6);
|
constructedFilters.push(`createdAt > ${date}`);
|
||||||
// Try to parse the timestamp as JavaScript Date
|
return false;
|
||||||
let date = Date.parse(timestamp);
|
} else if (term.startsWith("before:")) {
|
||||||
if (isNaN(date)) return false;
|
let timestamp = term.slice(7);
|
||||||
constructedFilters.push(`createdAt > ${date}`)
|
// Try to parse the timestamp as JavaScript Date
|
||||||
return false;
|
let date = Date.parse(timestamp);
|
||||||
} else if (term.startsWith("before:")) {
|
if (isNaN(date)) return false;
|
||||||
let timestamp = term.slice(7);
|
constructedFilters.push(`createdAt < ${date}`);
|
||||||
// Try to parse the timestamp as JavaScript Date
|
return false;
|
||||||
let date = Date.parse(timestamp);
|
|
||||||
if (isNaN(date)) return false;
|
|
||||||
constructedFilters.push(`createdAt < ${date}`)
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
})
|
|
||||||
|
|
||||||
logger.info(`Searching for ${splitSearch.join(" ")}`);
|
|
||||||
logger.info(`Limit: ${limit}`);
|
|
||||||
logger.info(`Offset: ${offset}`);
|
|
||||||
logger.info(`Filters: ${constructedFilters}`)
|
|
||||||
|
|
||||||
|
|
||||||
return posts.search(splitSearch.join(" "), {
|
|
||||||
limit: limit,
|
|
||||||
offset: offset,
|
|
||||||
filter: constructedFilters
|
|
||||||
});
|
|
||||||
},
|
|
||||||
ingestNote: async (ingestNotes: Note | Note[]) => {
|
|
||||||
if (ingestNotes instanceof Note) {
|
|
||||||
ingestNotes = [ingestNotes];
|
|
||||||
}
|
|
||||||
|
|
||||||
let indexingBatch: MeilisearchNote[] = [];
|
|
||||||
|
|
||||||
for (let note of ingestNotes) {
|
|
||||||
if (note.user === undefined) {
|
|
||||||
let user = await Users.findOne({
|
|
||||||
where: {
|
|
||||||
id: note.userId
|
|
||||||
}
|
|
||||||
});
|
|
||||||
note.user = user;
|
|
||||||
}
|
|
||||||
|
|
||||||
let attachmentType = "";
|
|
||||||
if (note.attachedFileTypes.length > 0) {
|
|
||||||
attachmentType = note.attachedFileTypes[0].split("/")[0];
|
|
||||||
switch (attachmentType) {
|
|
||||||
case "image":
|
|
||||||
case "video":
|
|
||||||
case "audio":
|
|
||||||
case "text":
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
attachmentType = "file"
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info(`Searching for ${splitSearch.join(" ")}`);
|
||||||
|
logger.info(`Limit: ${limit}`);
|
||||||
|
logger.info(`Offset: ${offset}`);
|
||||||
|
logger.info(`Filters: ${constructedFilters}`);
|
||||||
|
|
||||||
|
return posts.search(splitSearch.join(" "), {
|
||||||
|
limit: limit,
|
||||||
|
offset: offset,
|
||||||
|
filter: constructedFilters,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
ingestNote: async (ingestNotes: Note | Note[]) => {
|
||||||
|
if (ingestNotes instanceof Note) {
|
||||||
|
ingestNotes = [ingestNotes];
|
||||||
}
|
}
|
||||||
|
|
||||||
indexingBatch.push(<MeilisearchNote>{
|
let indexingBatch: MeilisearchNote[] = [];
|
||||||
|
|
||||||
|
for (let note of ingestNotes) {
|
||||||
|
if (note.user === undefined) {
|
||||||
|
let user = await Users.findOne({
|
||||||
|
where: {
|
||||||
|
id: note.userId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
note.user = user;
|
||||||
|
}
|
||||||
|
|
||||||
|
let attachmentType = "";
|
||||||
|
if (note.attachedFileTypes.length > 0) {
|
||||||
|
attachmentType = note.attachedFileTypes[0].split("/")[0];
|
||||||
|
switch (attachmentType) {
|
||||||
|
case "image":
|
||||||
|
case "video":
|
||||||
|
case "audio":
|
||||||
|
case "text":
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
attachmentType = "file";
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
indexingBatch.push(<MeilisearchNote>{
|
||||||
id: note.id.toString(),
|
id: note.id.toString(),
|
||||||
text: note.text ? note.text : "",
|
text: note.text ? note.text : "",
|
||||||
userId: note.userId,
|
userId: note.userId,
|
||||||
userHost: note.userHost !== "" ? note.userHost : url.parse(config.host).host,
|
userHost:
|
||||||
|
note.userHost !== ""
|
||||||
|
? note.userHost
|
||||||
|
: url.parse(config.host).host,
|
||||||
channelId: note.channelId ? note.channelId : "",
|
channelId: note.channelId ? note.channelId : "",
|
||||||
mediaAttachment: attachmentType,
|
mediaAttachment: attachmentType,
|
||||||
userName: note.user?.username ?? "UNKNOWN",
|
userName: note.user?.username ?? "UNKNOWN",
|
||||||
createdAt: note.createdAt.getTime() / 1000 // division by 1000 is necessary because Node returns in ms-accuracy
|
createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy
|
||||||
}
|
});
|
||||||
)
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let indexingIDs = indexingBatch.map(note => note.id);
|
let indexingIDs = indexingBatch.map((note) => note.id);
|
||||||
|
|
||||||
return posts.addDocuments(indexingBatch, {
|
return posts.addDocuments(indexingBatch, {
|
||||||
primaryKey: "id"
|
primaryKey: "id",
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
serverStats: async () => {
|
serverStats: async () => {
|
||||||
let health : Health = await client.health();
|
let health: Health = await client.health();
|
||||||
let stats: Stats = await client.getStats();
|
let stats: Stats = await client.getStats();
|
||||||
|
|
||||||
return {
|
return {
|
||||||
health: health.status,
|
health: health.status,
|
||||||
size: stats.databaseSize,
|
size: stats.databaseSize,
|
||||||
indexed_count: stats.indexes["posts"].numberOfDocuments
|
indexed_count: stats.indexes["posts"].numberOfDocuments,
|
||||||
}
|
};
|
||||||
|
},
|
||||||
}
|
}
|
||||||
} : null;
|
: null;
|
||||||
|
|
|
@ -3,7 +3,7 @@ import type Bull from "bull";
|
||||||
import { queueLogger } from "../../logger.js";
|
import { queueLogger } from "../../logger.js";
|
||||||
import { Notes } from "@/models/index.js";
|
import { Notes } from "@/models/index.js";
|
||||||
import { MoreThan } from "typeorm";
|
import { MoreThan } from "typeorm";
|
||||||
import { index } from "@/services/note/create.js";
|
import {index} from "@/services/note/create.js";
|
||||||
import {Note} from "@/models/entities/note.js";
|
import {Note} from "@/models/entities/note.js";
|
||||||
import meilisearch from "../../../db/meilisearch.js";
|
import meilisearch from "../../../db/meilisearch.js";
|
||||||
|
|
||||||
|
@ -33,13 +33,13 @@ export default async function indexAllNotes(
|
||||||
try {
|
try {
|
||||||
notes = await Notes.find({
|
notes = await Notes.find({
|
||||||
where: {
|
where: {
|
||||||
...(cursor ? { id: MoreThan(cursor) } : {}),
|
...(cursor ? {id: MoreThan(cursor)} : {}),
|
||||||
},
|
},
|
||||||
take: take,
|
take: take,
|
||||||
order: {
|
order: {
|
||||||
id: 1,
|
id: 1,
|
||||||
},
|
},
|
||||||
relations: ["user"]
|
relations: ["user"],
|
||||||
});
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.error(`Failed to query notes ${e}`);
|
logger.error(`Failed to query notes ${e}`);
|
||||||
|
@ -62,7 +62,7 @@ export default async function indexAllNotes(
|
||||||
const chunk = notes.slice(i, i + batch);
|
const chunk = notes.slice(i, i + batch);
|
||||||
|
|
||||||
if (meilisearch) {
|
if (meilisearch) {
|
||||||
await meilisearch.ingestNote(chunk)
|
await meilisearch.ingestNote(chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
await Promise.all(chunk.map((note) => index(note, true)));
|
await Promise.all(chunk.map((note) => index(note, true)));
|
||||||
|
|
|
@ -172,7 +172,7 @@ export default define(meta, paramDef, async (ps, me) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
return found;
|
return found;
|
||||||
} else if(meilisearch) {
|
} else if (meilisearch) {
|
||||||
let start = 0;
|
let start = 0;
|
||||||
const chunkSize = 100;
|
const chunkSize = 100;
|
||||||
|
|
||||||
|
@ -236,7 +236,6 @@ export default define(meta, paramDef, async (ps, me) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
return found;
|
return found;
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
const userQuery =
|
const userQuery =
|
||||||
ps.userId != null
|
ps.userId != null
|
||||||
|
|
|
@ -34,8 +34,7 @@ export default define(meta, paramDef, async () => {
|
||||||
total: fsStats[0].size,
|
total: fsStats[0].size,
|
||||||
used: fsStats[0].used,
|
used: fsStats[0].used,
|
||||||
},
|
},
|
||||||
meilisearch: meilisearchStats
|
meilisearch: meilisearchStats,
|
||||||
|
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -6,12 +6,13 @@ export async function search() {
|
||||||
const { canceled, result: query } = await os.inputText({
|
const { canceled, result: query } = await os.inputText({
|
||||||
title: i18n.ts.search,
|
title: i18n.ts.search,
|
||||||
placeholder: "Enter search terms...",
|
placeholder: "Enter search terms...",
|
||||||
text: "Advanced search operators\n" +
|
text:
|
||||||
|
"Advanced search operators\n" +
|
||||||
"from:user => filter by user\n" +
|
"from:user => filter by user\n" +
|
||||||
"has:image/video/audio/text/file => filter by attachment types\n" +
|
"has:image/video/audio/text/file => filter by attachment types\n" +
|
||||||
"domain:domain.com => filter by domain\n" +
|
"domain:domain.com => filter by domain\n" +
|
||||||
"before:Date => show posts made before Date\n" +
|
"before:Date => show posts made before Date\n" +
|
||||||
"after:Date => show posts made after Date"
|
"after:Date => show posts made after Date",
|
||||||
});
|
});
|
||||||
if (canceled || query == null || query === "") return;
|
if (canceled || query == null || query === "") return;
|
||||||
|
|
||||||
|
|
|
@ -61,7 +61,7 @@ import XNet from "./net.vue";
|
||||||
import XCpu from "./cpu.vue";
|
import XCpu from "./cpu.vue";
|
||||||
import XMemory from "./mem.vue";
|
import XMemory from "./mem.vue";
|
||||||
import XDisk from "./disk.vue";
|
import XDisk from "./disk.vue";
|
||||||
import XMeili from "./meilisearch.vue"
|
import XMeili from "./meilisearch.vue";
|
||||||
import MkContainer from "@/components/MkContainer.vue";
|
import MkContainer from "@/components/MkContainer.vue";
|
||||||
import { GetFormResultType } from "@/scripts/form";
|
import { GetFormResultType } from "@/scripts/form";
|
||||||
import * as os from "@/os";
|
import * as os from "@/os";
|
||||||
|
|
Loading…
Reference in New Issue