natty
/
calckey
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: formatting

This commit is contained in:
ThatOneCalculator 2023-05-28 20:34:18 -07:00
parent 5c56487802
commit 2cf1d9f630
No known key found for this signature in database
GPG Key ID: 8703CACD01000000
6 changed files with 192 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
locales/en-US.yml
View File

@ -5,7 +5,7 @@ introMisskey: "Welcome! Calckey is an open source, decentralized social media pl
\ that's free forever! \U0001F680" \ that's free forever! \U0001F680"
monthAndDay: "{month}/{day}" monthAndDay: "{month}/{day}"
search: "Search" search: "Search"
search_placeholder: "Enter search terms..." searchPlaceholder: "Search Calckey"
notifications: "Notifications" notifications: "Notifications"
username: "Username" username: "Username"
password: "Password" password: "Password"

360
packages/backend/src/db/meilisearch.ts
View File

@ -1,11 +1,11 @@
import {Health, MeiliSearch, Stats} from "meilisearch"; import { Health, MeiliSearch, Stats } from "meilisearch";
import {dbLogger} from "./logger.js"; import { dbLogger } from "./logger.js";
import config from "@/config/index.js"; import config from "@/config/index.js";
import {Note} from "@/models/entities/note.js"; import { Note } from "@/models/entities/note.js";
import * as url from "url"; import * as url from "url";
import {ILocalUser, User} from "@/models/entities/user.js"; import { ILocalUser, User } from "@/models/entities/user.js";
import {Followings, Users} from "@/models/index.js"; import { Followings, Users } from "@/models/index.js";
const logger = dbLogger.createSubLogger("meilisearch", "gray", false); const logger = dbLogger.createSubLogger("meilisearch", "gray", false);
@ -72,195 +72,195 @@ export type MeilisearchNote = {
export default hasConfig export default hasConfig
? { ? {
search: async ( search: async (
query: string, query: string,
limit: number, limit: number,
offset: number, offset: number,
userCtx: ILocalUser | null, userCtx: ILocalUser | null,
) => { ) => {
/// Advanced search syntax /// Advanced search syntax
/// from:user => filter by user + optional domain /// from:user => filter by user + optional domain
/// has:image/video/audio/text/file => filter by attachment types /// has:image/video/audio/text/file => filter by attachment types
/// domain:domain.com => filter by domain /// domain:domain.com => filter by domain
/// before:Date => show posts made before Date /// before:Date => show posts made before Date
/// after: Date => show posts made after Date /// after: Date => show posts made after Date
/// "text" => get posts with exact text between quotes /// "text" => get posts with exact text between quotes
/// filter:following => show results only from users you follow /// filter:following => show results only from users you follow
/// filter:followers => show results only from followers /// filter:followers => show results only from followers
let constructedFilters: string[] = []; let constructedFilters: string[] = [];
let splitSearch = query.split(" "); let splitSearch = query.split(" ");
// Detect search operators and remove them from the actual query // Detect search operators and remove them from the actual query
let filteredSearchTerms = ( let filteredSearchTerms = (
await Promise.all( await Promise.all(
splitSearch.map(async (term) => { splitSearch.map(async (term) => {
if (term.startsWith("has:")) { if (term.startsWith("has:")) {
let fileType = term.slice(4); let fileType = term.slice(4);
constructedFilters.push(`mediaAttachment = "${fileType}"`); constructedFilters.push(`mediaAttachment = "${fileType}"`);
return null; return null;
} else if (term.startsWith("from:")) { } else if (term.startsWith("from:")) {
let user = term.slice(5); let user = term.slice(5);
constructedFilters.push(`userName = ${user}`); constructedFilters.push(`userName = ${user}`);
return null; return null;
} else if (term.startsWith("domain:")) { } else if (term.startsWith("domain:")) {
let domain = term.slice(7); let domain = term.slice(7);
constructedFilters.push(`userHost = ${domain}`); constructedFilters.push(`userHost = ${domain}`);
return null; return null;
} else if (term.startsWith("after:")) { } else if (term.startsWith("after:")) {
let timestamp = term.slice(6); let timestamp = term.slice(6);
// Try to parse the timestamp as JavaScript Date // Try to parse the timestamp as JavaScript Date
let date = Date.parse(timestamp); let date = Date.parse(timestamp);
if (isNaN(date)) return null; if (isNaN(date)) return null;
constructedFilters.push(`createdAt > ${date / 1000}`); constructedFilters.push(`createdAt > ${date / 1000}`);
return null; return null;
} else if (term.startsWith("before:")) { } else if (term.startsWith("before:")) {
let timestamp = term.slice(7); let timestamp = term.slice(7);
// Try to parse the timestamp as JavaScript Date // Try to parse the timestamp as JavaScript Date
let date = Date.parse(timestamp); let date = Date.parse(timestamp);
if (isNaN(date)) return null; if (isNaN(date)) return null;
constructedFilters.push(`createdAt < ${date / 1000}`); constructedFilters.push(`createdAt < ${date / 1000}`);
return null; return null;
} else if (term.startsWith("filter:following")) { } else if (term.startsWith("filter:following")) {
// Check if we got a context user // Check if we got a context user
if (userCtx) { if (userCtx) {
// Fetch user follows from DB // Fetch user follows from DB
let followedUsers = await Followings.find({ let followedUsers = await Followings.find({
where: { where: {
followerId: userCtx.id, followerId: userCtx.id,
}, },
select: { select: {
followeeId: true, followeeId: true,
}, },
}); });
let followIDs = followedUsers.map((user) => user.followeeId); let followIDs = followedUsers.map((user) => user.followeeId);
if (followIDs.length === 0) return null; if (followIDs.length === 0) return null;
constructedFilters.push(`userId IN [${followIDs.join(",")}]`); constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
} else { } else {
logger.warn( logger.warn(
"search filtered to follows called without user context", "search filtered to follows called without user context",
); );
}
return null;
} else if (term.startsWith("filter:followers")) {
// Check if we got a context user
if (userCtx) {
// Fetch users follows from DB
let followedUsers = await Followings.find({
where: {
followeeId: userCtx.id,
},
select: {
followerId: true,
},
});
let followIDs = followedUsers.map((user) => user.followerId);
if (followIDs.length === 0) return null;
constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
} else {
logger.warn(
"search filtered to followers called without user context",
);
}
return null;
} }
return null; return term;
} else if (term.startsWith("filter:followers")) { }),
// Check if we got a context user )
if (userCtx) { ).filter((term) => term !== null);
// Fetch users follows from DB
let followedUsers = await Followings.find({
where: {
followeeId: userCtx.id,
},
select: {
followerId: true,
},
});
let followIDs = followedUsers.map((user) => user.followerId);
if (followIDs.length === 0) return null; let sortRules = [];
constructedFilters.push(`userId IN [${followIDs.join(",")}]`); // An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search
} else { // These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want
logger.warn( if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) {
"search filtered to followers called without user context", sortRules.push("createdAt:desc");
); }
}
return null; logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
logger.info(`Limit: ${limit}`);
logger.info(`Offset: ${offset}`);
logger.info(`Filters: ${constructedFilters}`);
logger.info(`Ordering: ${sortRules}`);
return posts.search(filteredSearchTerms.join(" "), {
limit: limit,
offset: offset,
filter: constructedFilters,
sort: sortRules,
});
},
ingestNote: async (ingestNotes: Note | Note[]) => {
if (ingestNotes instanceof Note) {
ingestNotes = [ingestNotes];
}
let indexingBatch: MeilisearchNote[] = [];
for (let note of ingestNotes) {
if (note.user === undefined) {
note.user = await Users.findOne({
where: {
id: note.userId,
},
});
}
let attachmentType = "";
if (note.attachedFileTypes.length > 0) {
attachmentType = note.attachedFileTypes[0].split("/")[0];
switch (attachmentType) {
case "image":
case "video":
case "audio":
case "text":
break;
default:
attachmentType = "file";
break;
} }
}
return term; indexingBatch.push(<MeilisearchNote>{
}), id: note.id.toString(),
) text: note.text ? note.text : "",
).filter((term) => term !== null); userId: note.userId,
userHost:
let sortRules = []; note.userHost !== ""
? note.userHost
// An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search : url.parse(config.host).host,
// These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want channelId: note.channelId ? note.channelId : "",
if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) { mediaAttachment: attachmentType,
sortRules.push("createdAt:desc"); userName: note.user?.username ?? "UNKNOWN",
} createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy
logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
logger.info(`Limit: ${limit}`);
logger.info(`Offset: ${offset}`);
logger.info(`Filters: ${constructedFilters}`);
logger.info(`Ordering: ${sortRules}`);
return posts.search(filteredSearchTerms.join(" "), {
limit: limit,
offset: offset,
filter: constructedFilters,
sort: sortRules,
});
},
ingestNote: async (ingestNotes: Note | Note[]) => {
if (ingestNotes instanceof Note) {
ingestNotes = [ingestNotes];
}
let indexingBatch: MeilisearchNote[] = [];
for (let note of ingestNotes) {
if (note.user === undefined) {
note.user = await Users.findOne({
where: {
id: note.userId,
},
}); });
} }
let attachmentType = ""; return posts
if (note.attachedFileTypes.length > 0) { .addDocuments(indexingBatch, {
attachmentType = note.attachedFileTypes[0].split("/")[0]; primaryKey: "id",
switch (attachmentType) { })
case "image": .then(() =>
case "video": console.log(`sent ${indexingBatch.length} posts for indexing`),
case "audio": );
case "text": },
break; serverStats: async () => {
default: let health: Health = await client.health();
attachmentType = "file"; let stats: Stats = await client.getStats();
break;
}
}
indexingBatch.push(<MeilisearchNote>{ return {
id: note.id.toString(), health: health.status,
text: note.text ? note.text : "", size: stats.databaseSize,
userId: note.userId, indexed_count: stats.indexes["posts"].numberOfDocuments,
userHost: };
note.userHost !== "" },
? note.userHost }
: url.parse(config.host).host,
channelId: note.channelId ? note.channelId : "",
mediaAttachment: attachmentType,
userName: note.user?.username ?? "UNKNOWN",
createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy
});
}
return posts
.addDocuments(indexingBatch, {
primaryKey: "id",
})
.then(() =>
console.log(`sent ${indexingBatch.length} posts for indexing`),
);
},
serverStats: async () => {
let health: Health = await client.health();
let stats: Stats = await client.getStats();
return {
health: health.status,
size: stats.databaseSize,
indexed_count: stats.indexes["posts"].numberOfDocuments,
};
},
}
: null; : null;

14
packages/backend/src/queue/processors/background/index-all-notes.ts
View File

@ -1,10 +1,10 @@
import type Bull from "bull"; import type Bull from "bull";
import {queueLogger} from "../../logger.js"; import { queueLogger } from "../../logger.js";
import {Notes} from "@/models/index.js"; import { Notes } from "@/models/index.js";
import {MoreThan} from "typeorm"; import { MoreThan } from "typeorm";
import {index} from "@/services/note/create.js"; import { index } from "@/services/note/create.js";
import {Note} from "@/models/entities/note.js"; import { Note } from "@/models/entities/note.js";
import meilisearch from "../../../db/meilisearch.js"; import meilisearch from "../../../db/meilisearch.js";
const logger = queueLogger.createSubLogger("index-all-notes"); const logger = queueLogger.createSubLogger("index-all-notes");
@ -33,7 +33,7 @@ export default async function indexAllNotes(
try { try {
notes = await Notes.find({ notes = await Notes.find({
where: { where: {
...(cursor ? {id: MoreThan(cursor)} : {}), ...(cursor ? { id: MoreThan(cursor) } : {}),
}, },
take: take, take: take,
order: { order: {
@ -69,7 +69,7 @@ export default async function indexAllNotes(
indexedCount += chunk.length; indexedCount += chunk.length;
const pct = (indexedCount / total) * 100; const pct = (indexedCount / total) * 100;
job.update({indexedCount, cursor, total}); job.update({ indexedCount, cursor, total });
job.progress(+pct.toFixed(1)); job.progress(+pct.toFixed(1));
logger.info(`Indexed notes ${indexedCount}/${total ? total : "?"}`); logger.info(`Indexed notes ${indexedCount}/${total ? total : "?"}`);
} }

2
packages/backend/src/server/api/endpoints/notes/search.ts
View File

@ -4,7 +4,7 @@ import { Note } from "@/models/entities/note.js";
import config from "@/config/index.js"; import config from "@/config/index.js";
import es from "../../../../db/elasticsearch.js"; import es from "../../../../db/elasticsearch.js";
import sonic from "../../../../db/sonic.js"; import sonic from "../../../../db/sonic.js";
import meilisearch, {MeilisearchNote} from "../../../../db/meilisearch.js"; import meilisearch, { MeilisearchNote } from "../../../../db/meilisearch.js";
import define from "../../define.js"; import define from "../../define.js";
import { makePaginationQuery } from "../../common/make-pagination-query.js"; import { makePaginationQuery } from "../../common/make-pagination-query.js";
import { generateVisibilityQuery } from "../../common/generate-visibility-query.js"; import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";

2
packages/client/src/scripts/search.ts
View File

@ -5,7 +5,7 @@ import { mainRouter } from "@/router";
export async function search() { export async function search() {
const { canceled, result: query } = await os.inputText({ const { canceled, result: query } = await os.inputText({
title: i18n.ts.search, title: i18n.ts.search,
placeholder: i18n.ts.search_placeholder, placeholder: i18n.ts.searchPlaceholder,
text: text:
"Advanced search operators\n" + "Advanced search operators\n" +
"from:user => filter by user\n" + "from:user => filter by user\n" +

4
packages/client/src/widgets/server-metric/meilisearch.vue
View File

@ -11,9 +11,9 @@
</template> </template>
<script lang="ts" setup> <script lang="ts" setup>
import {onBeforeUnmount, onMounted} from "vue"; import { onBeforeUnmount, onMounted } from "vue";
import bytes from "@/filters/bytes"; import bytes from "@/filters/bytes";
import {i18n} from "@/i18n"; import { i18n } from "@/i18n";
const props = defineProps<{ const props = defineProps<{
connection: any; connection: any;