From 5fd66907558b5124bc555b957c18631e0299548a Mon Sep 17 00:00:00 2001
From: April John <waterdev@galaxycrow.de>
Date: Fri, 5 May 2023 14:30:08 +0200
Subject: [PATCH 1/4] Fix: properly handle timeline db errors

---
 .../api/endpoints/notes/global-timeline.ts    | 19 +++++++++++----
 .../api/endpoints/notes/hybrid-timeline.ts    | 19 +++++++++++----
 .../api/endpoints/notes/local-timeline.ts     | 19 +++++++++++----
 .../endpoints/notes/recommended-timeline.ts   | 19 +++++++++++----
 .../server/api/endpoints/notes/timeline.ts    | 23 +++++++++++++++----
 .../api/endpoints/notes/user-list-timeline.ts | 19 +++++++++++----
 6 files changed, 88 insertions(+), 30 deletions(-)

diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
index 077a1ad5e1..93fbda8c76 100644
--- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
@@ -32,6 +32,11 @@ export const meta = {
 			code: "GTL_DISABLED",
 			id: "0332fc13-6ab2-4427-ae80-a9fadffd1a6b",
 		},
+		queryError: {
+			message: "Please follow more users.",
+			code: "QUERY_ERROR",
+			id: "620763f4-f621-4533-ab33-0577a1a3c343",
+		},
 	},
 } as const;
 
@@ -106,11 +111,15 @@ export default define(meta, paramDef, async (ps, user) => {
 	const found = [];
 	const take = Math.floor(ps.limit * 1.5);
 	let skip = 0;
-	while (found.length < ps.limit) {
-		const notes = await query.take(take).skip(skip).getMany();
-		found.push(...(await Notes.packMany(notes, user)));
-		skip += take;
-		if (notes.length < take) break;
+	try {
+		while (found.length < ps.limit) {
+			const notes = await query.take(take).skip(skip).getMany();
+			found.push(...(await Notes.packMany(notes, user)));
+			skip += take;
+			if (notes.length < take) break;
+		}
+	} catch (error) {
+		throw new ApiError(meta.errors.queryError);
 	}
 
 	if (found.length > ps.limit) {
diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
index 3c171278b4..06f9edf453 100644
--- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
@@ -36,6 +36,11 @@ export const meta = {
 			code: "STL_DISABLED",
 			id: "620763f4-f621-4533-ab33-0577a1a3c342",
 		},
+		queryError: {
+			message: "Please follow more users.",
+			code: "QUERY_ERROR",
+			id: "620763f4-f621-4533-ab33-0577a1a3c343",
+		},
 	},
 } as const;
 
@@ -162,11 +167,15 @@ export default define(meta, paramDef, async (ps, user) => {
 	const found = [];
 	const take = Math.floor(ps.limit * 1.5);
 	let skip = 0;
-	while (found.length < ps.limit) {
-		const notes = await query.take(take).skip(skip).getMany();
-		found.push(...(await Notes.packMany(notes, user)));
-		skip += take;
-		if (notes.length < take) break;
+	try {
+		while (found.length < ps.limit) {
+			const notes = await query.take(take).skip(skip).getMany();
+			found.push(...(await Notes.packMany(notes, user)));
+			skip += take;
+			if (notes.length < take) break;
+		}
+	} catch (error) {
+		throw new ApiError(meta.errors.queryError);
 	}
 
 	if (found.length > ps.limit) {
diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
index cec371c8dc..ec3d4bde09 100644
--- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
@@ -35,6 +35,11 @@ export const meta = {
 			code: "LTL_DISABLED",
 			id: "45a6eb02-7695-4393-b023-dd3be9aaaefd",
 		},
+		queryError: {
+			message: "Please follow more users.",
+			code: "QUERY_ERROR",
+			id: "620763f4-f621-4533-ab33-0577a1a3c343",
+		},
 	},
 } as const;
 
@@ -136,11 +141,15 @@ export default define(meta, paramDef, async (ps, user) => {
 	const found = [];
 	const take = Math.floor(ps.limit * 1.5);
 	let skip = 0;
-	while (found.length < ps.limit) {
-		const notes = await query.take(take).skip(skip).getMany();
-		found.push(...(await Notes.packMany(notes, user)));
-		skip += take;
-		if (notes.length < take) break;
+	try {
+		while (found.length < ps.limit) {
+			const notes = await query.take(take).skip(skip).getMany();
+			found.push(...(await Notes.packMany(notes, user)));
+			skip += take;
+			if (notes.length < take) break;
+		}
+	} catch (error) {
+		throw new ApiError(meta.errors.queryError);
 	}
 
 	if (found.length > ps.limit) {
diff --git a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
index 56847b1dd2..27b72f4a3f 100644
--- a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
@@ -35,6 +35,11 @@ export const meta = {
 			code: "RTL_DISABLED",
 			id: "45a6eb02-7695-4393-b023-dd3be9aaaefe",
 		},
+		queryError: {
+			message: "Please follow more users.",
+			code: "QUERY_ERROR",
+			id: "620763f4-f621-4533-ab33-0577a1a3c343",
+		},
 	},
 } as const;
 
@@ -139,11 +144,15 @@ export default define(meta, paramDef, async (ps, user) => {
 	const found = [];
 	const take = Math.floor(ps.limit * 1.5);
 	let skip = 0;
-	while (found.length < ps.limit) {
-		const notes = await query.take(take).skip(skip).getMany();
-		found.push(...(await Notes.packMany(notes, user)));
-		skip += take;
-		if (notes.length < take) break;
+	try {
+		while (found.length < ps.limit) {
+			const notes = await query.take(take).skip(skip).getMany();
+			found.push(...(await Notes.packMany(notes, user)));
+			skip += take;
+			if (notes.length < take) break;
+		}
+	} catch (error) {
+		throw new ApiError(meta.errors.queryError);
 	}
 
 	if (found.length > ps.limit) {
diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts
index f85c0cfd32..436b50e9f4 100644
--- a/packages/backend/src/server/api/endpoints/notes/timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts
@@ -10,6 +10,7 @@ import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.j
 import { generateChannelQuery } from "../../common/generate-channel-query.js";
 import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
 import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
+import { ApiError } from "../../error.js";
 
 export const meta = {
 	tags: ["notes"],
@@ -27,6 +28,14 @@ export const meta = {
 			ref: "Note",
 		},
 	},
+
+	errors: {
+		queryError: {
+			message: "Please follow more users.",
+			code: "QUERY_ERROR",
+			id: "620763f4-f621-4533-ab33-0577a1a3c343",
+		},
+	},
 } as const;
 
 export const paramDef = {
@@ -154,11 +163,15 @@ export default define(meta, paramDef, async (ps, user) => {
 	const found = [];
 	const take = Math.floor(ps.limit * 1.5);
 	let skip = 0;
-	while (found.length < ps.limit) {
-		const notes = await query.take(take).skip(skip).getMany();
-		found.push(...(await Notes.packMany(notes, user)));
-		skip += take;
-		if (notes.length < take) break;
+	try {
+		while (found.length < ps.limit) {
+			const notes = await query.take(take).skip(skip).getMany();
+			found.push(...(await Notes.packMany(notes, user)));
+			skip += take;
+			if (notes.length < take) break;
+		}
+	} catch (error) {
+		throw new ApiError(meta.errors.queryError);
 	}
 
 	if (found.length > ps.limit) {
diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts
index 03f5cee3f3..5c3fc55bef 100644
--- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts
@@ -29,6 +29,11 @@ export const meta = {
 			code: "NO_SUCH_LIST",
 			id: "8fb1fbd5-e476-4c37-9fb0-43d55b63a2ff",
 		},
+		queryError: {
+			message: "Please follow more users.",
+			code: "QUERY_ERROR",
+			id: "620763f4-f621-4533-ab33-0577a1a3c343",
+		},
 	},
 } as const;
 
@@ -149,11 +154,15 @@ export default define(meta, paramDef, async (ps, user) => {
 	const found = [];
 	const take = Math.floor(ps.limit * 1.5);
 	let skip = 0;
-	while (found.length < ps.limit) {
-		const notes = await query.take(take).skip(skip).getMany();
-		found.push(...(await Notes.packMany(notes, user)));
-		skip += take;
-		if (notes.length < take) break;
+	try {
+		while (found.length < ps.limit) {
+			const notes = await query.take(take).skip(skip).getMany();
+			found.push(...(await Notes.packMany(notes, user)));
+			skip += take;
+			if (notes.length < take) break;
+		}
+	} catch (error) {
+		throw new ApiError(meta.errors.queryError);
 	}
 
 	if (found.length > ps.limit) {

From 3800eb8980eee8e446038355b8faf29554699825 Mon Sep 17 00:00:00 2001
From: April John <waterdev@galaxycrow.de>
Date: Fri, 5 May 2023 15:16:10 +0200
Subject: [PATCH 2/4] Feat: new internal post visibility "hidden"

---
 .../migration/1682891891317-AddHiddenPosts.js       | 13 +++++++++++++
 packages/backend/src/models/entities/note.ts        |  1 +
 .../backend/src/queue/processors/db/import-posts.ts |  4 ++--
 .../backend/src/remote/activitypub/models/note.ts   |  4 ++--
 .../server/api/endpoints/notes/global-timeline.ts   |  1 +
 .../server/api/endpoints/notes/hybrid-timeline.ts   |  2 ++
 .../server/api/endpoints/notes/local-timeline.ts    |  1 +
 .../api/endpoints/notes/recommended-timeline.ts     |  1 +
 .../src/server/api/endpoints/notes/timeline.ts      |  2 ++
 .../backend/src/services/note/reaction/create.ts    |  2 +-
 packages/backend/src/types.ts                       |  1 +
 11 files changed, 27 insertions(+), 5 deletions(-)
 create mode 100644 packages/backend/migration/1682891891317-AddHiddenPosts.js

diff --git a/packages/backend/migration/1682891891317-AddHiddenPosts.js b/packages/backend/migration/1682891891317-AddHiddenPosts.js
new file mode 100644
index 0000000000..9fbb553b3b
--- /dev/null
+++ b/packages/backend/migration/1682891891317-AddHiddenPosts.js
@@ -0,0 +1,13 @@
+export class AddHiddenPosts1682891891317 {
+	name = "AddHiddenPosts1682891891317";
+
+	async up(queryRunner) {
+		await queryRunner.query(
+			`ALTER TYPE note_visibility_enum ADD VALUE IF NOT EXISTS 'hidden'`,
+		);
+	}
+
+	async down(queryRunner) {
+		await queryRunner.query(`ALTER TYPE note_visibility_enum REMOVE VALUE IF EXISTS 'hidden'`);
+	}
+}
diff --git a/packages/backend/src/models/entities/note.ts b/packages/backend/src/models/entities/note.ts
index 10449bb6d6..f4e76c1dba 100644
--- a/packages/backend/src/models/entities/note.ts
+++ b/packages/backend/src/models/entities/note.ts
@@ -111,6 +111,7 @@ export class Note {
 	/**
 	 * public ... 公開
 	 * home ... ホームタイムライン(ユーザーページのタイムライン含む)のみに流す
+	 * hidden ... only visible on profile (doesnt federate, like local only, but can be fetched via AP like home) <- for now only used for post imports
 	 * followers ... フォロワーのみ
 	 * specified ... visibleUserIds で指定したユーザーのみ
 	 */
diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts
index 20ef3a518d..a0a916b41e 100644
--- a/packages/backend/src/queue/processors/db/import-posts.ts
+++ b/packages/backend/src/queue/processors/db/import-posts.ts
@@ -65,7 +65,7 @@ export async function importPosts(
 						renote: null,
 						cw: cw,
 						localOnly,
-						visibility: "public",
+						visibility: "hidden",
 						visibleUsers: [],
 						channel: null,
 						apMentions: new Array(0),
@@ -109,7 +109,7 @@ export async function importPosts(
 						renote: null,
 						cw: post.sensitive,
 						localOnly: false,
-						visibility: "public",
+						visibility: "hidden",
 						visibleUsers: [],
 						channel: null,
 						apMentions: new Array(0),
diff --git a/packages/backend/src/remote/activitypub/models/note.ts b/packages/backend/src/remote/activitypub/models/note.ts
index 7b41de6700..719feb56f6 100644
--- a/packages/backend/src/remote/activitypub/models/note.ts
+++ b/packages/backend/src/remote/activitypub/models/note.ts
@@ -686,7 +686,7 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) {
 				multiple: poll?.multiple,
 				votes: poll?.votes,
 				expiresAt: poll?.expiresAt,
-				noteVisibility: note.visibility,
+				noteVisibility: note.visibility === "hidden" ? "home" : note.visibility,
 				userId: actor.id,
 				userHost: actor.host,
 			});
@@ -704,7 +704,7 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) {
 					multiple: poll?.multiple,
 					votes: poll?.votes,
 					expiresAt: poll?.expiresAt,
-					noteVisibility: note.visibility,
+					noteVisibility: note.visibility === "hidden" ? "home" : note.visibility,
 				},
 			);
 			updating = true;
diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
index 93fbda8c76..78a1932830 100644
--- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
@@ -98,6 +98,7 @@ export default define(meta, paramDef, async (ps, user) => {
 	if (ps.withFiles) {
 		query.andWhere("note.fileIds != '{}'");
 	}
+	query.andWhere("note.visibility != 'hidden'");
 	//#endregion
 
 	process.nextTick(() => {
diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
index 06f9edf453..508b268cc0 100644
--- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
@@ -156,6 +156,8 @@ export default define(meta, paramDef, async (ps, user) => {
 	if (ps.withFiles) {
 		query.andWhere("note.fileIds != '{}'");
 	}
+
+	query.andWhere("note.visibility != 'hidden'");
 	//#endregion
 
 	process.nextTick(() => {
diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
index ec3d4bde09..797c6d77cb 100644
--- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
@@ -128,6 +128,7 @@ export default define(meta, paramDef, async (ps, user) => {
 			);
 		}
 	}
+	query.andWhere("note.visibility != 'hidden'");
 	//#endregion
 
 	process.nextTick(() => {
diff --git a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
index 27b72f4a3f..321ab4ad7d 100644
--- a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
@@ -131,6 +131,7 @@ export default define(meta, paramDef, async (ps, user) => {
 			);
 		}
 	}
+	query.andWhere("note.visibility != 'hidden'");
 	//#endregion
 
 	process.nextTick(() => {
diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts
index 436b50e9f4..62996efdde 100644
--- a/packages/backend/src/server/api/endpoints/notes/timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts
@@ -152,6 +152,8 @@ export default define(meta, paramDef, async (ps, user) => {
 	if (ps.withFiles) {
 		query.andWhere("note.fileIds != '{}'");
 	}
+
+	query.andWhere("note.visibility != 'hidden'");
 	//#endregion
 
 	process.nextTick(() => {
diff --git a/packages/backend/src/services/note/reaction/create.ts b/packages/backend/src/services/note/reaction/create.ts
index 277393eb41..4b9b20ec30 100644
--- a/packages/backend/src/services/note/reaction/create.ts
+++ b/packages/backend/src/services/note/reaction/create.ts
@@ -144,7 +144,7 @@ export default async (
 	});
 
 	//#region deliver
-	if (Users.isLocalUser(user) && !note.localOnly) {
+	if (Users.isLocalUser(user) && !note.localOnly && note.visibility !== "hidden") {
 		const content = renderActivity(await renderLike(record, note));
 		const dm = new DeliverManager(user, content);
 		if (note.userHost !== null) {
diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts
index 9e53440a17..2ba6da2f7d 100644
--- a/packages/backend/src/types.ts
+++ b/packages/backend/src/types.ts
@@ -18,6 +18,7 @@ export const noteVisibilities = [
 	"home",
 	"followers",
 	"specified",
+	"hidden",
 ] as const;
 
 export const mutedNoteReasons = ["word", "manual", "spam", "other"] as const;

From 6b6e5dfcdfe80f4f8836007e3cf8424ef8b90b13 Mon Sep 17 00:00:00 2001
From: April John <waterdev@galaxycrow.de>
Date: Fri, 5 May 2023 15:50:50 +0200
Subject: [PATCH 3/4] hide hidden posts in websocket streams

---
 packages/backend/src/server/api/stream/channels/channel.ts       | 1 +
 packages/backend/src/server/api/stream/channels/hashtag.ts       | 1 +
 .../backend/src/server/api/stream/channels/hybrid-timeline.ts    | 1 +
 3 files changed, 3 insertions(+)

diff --git a/packages/backend/src/server/api/stream/channels/channel.ts b/packages/backend/src/server/api/stream/channels/channel.ts
index d046579f42..aa3c8528fb 100644
--- a/packages/backend/src/server/api/stream/channels/channel.ts
+++ b/packages/backend/src/server/api/stream/channels/channel.ts
@@ -29,6 +29,7 @@ export default class extends Channel {
 	}
 
 	private async onNote(note: Packed<"Note">) {
+		if (note.visibility === "hidden") return;
 		if (note.channelId !== this.channelId) return;
 
 		// 流れてきたNoteがミュートしているユーザーが関わるものだったら無視する
diff --git a/packages/backend/src/server/api/stream/channels/hashtag.ts b/packages/backend/src/server/api/stream/channels/hashtag.ts
index a2e5481abb..f1ba0a9dc7 100644
--- a/packages/backend/src/server/api/stream/channels/hashtag.ts
+++ b/packages/backend/src/server/api/stream/channels/hashtag.ts
@@ -24,6 +24,7 @@ export default class extends Channel {
 	}
 
 	private async onNote(note: Packed<"Note">) {
+		if (note.visibility === "hidden") return;
 		const noteTags = note.tags
 			? note.tags.map((t: string) => t.toLowerCase())
 			: [];
diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts
index 557bb96827..2ac8b0fb79 100644
--- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts
+++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts
@@ -29,6 +29,7 @@ export default class extends Channel {
 	}
 
 	private async onNote(note: Packed<"Note">) {
+		if (note.visibility === "hidden") return;
 		// チャンネルの投稿ではなく、自分自身の投稿 または
 		// チャンネルの投稿ではなく、その投稿のユーザーをフォローしている または
 		// チャンネルの投稿ではなく、全体公開のローカルの投稿 または

From dd777935cb840d76ab09357c92f5145ad20c2b28 Mon Sep 17 00:00:00 2001
From: April John <waterdev@galaxycrow.de>
Date: Fri, 5 May 2023 15:53:05 +0200
Subject: [PATCH 4/4] fix: dont stream hidden posts over websocket

---
 packages/backend/src/server/api/stream/channels/home-timeline.ts | 1 +
 .../src/server/api/stream/channels/recommended-timeline.ts       | 1 +
 packages/backend/src/server/api/stream/channels/user-list.ts     | 1 +
 3 files changed, 3 insertions(+)

diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts
index fa4a8a3901..59501bafaf 100644
--- a/packages/backend/src/server/api/stream/channels/home-timeline.ts
+++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts
@@ -20,6 +20,7 @@ export default class extends Channel {
 	}
 
 	private async onNote(note: Packed<"Note">) {
+		if (note.visibility === "hidden") return;
 		if (note.channelId) {
 			if (!this.followingChannels.has(note.channelId)) return;
 		} else {
diff --git a/packages/backend/src/server/api/stream/channels/recommended-timeline.ts b/packages/backend/src/server/api/stream/channels/recommended-timeline.ts
index 6baec77442..37e29cd9a5 100644
--- a/packages/backend/src/server/api/stream/channels/recommended-timeline.ts
+++ b/packages/backend/src/server/api/stream/channels/recommended-timeline.ts
@@ -29,6 +29,7 @@ export default class extends Channel {
 	}
 
 	private async onNote(note: Packed<"Note">) {
+		if (note.visibility === "hidden") return;
 		// チャンネルの投稿ではなく、自分自身の投稿 または
 		// チャンネルの投稿ではなく、その投稿のユーザーをフォローしている または
 		// チャンネルの投稿ではなく、全体公開のローカルの投稿 または
diff --git a/packages/backend/src/server/api/stream/channels/user-list.ts b/packages/backend/src/server/api/stream/channels/user-list.ts
index 105c45955c..c2b62c05a7 100644
--- a/packages/backend/src/server/api/stream/channels/user-list.ts
+++ b/packages/backend/src/server/api/stream/channels/user-list.ts
@@ -49,6 +49,7 @@ export default class extends Channel {
 	}
 
 	private async onNote(note: Packed<"Note">) {
+		if (note.visibility === "hidden") return;
 		if (!this.listUsers.includes(note.userId)) return;
 
 		// 流れてきたNoteがミュートしているユーザーが関わるものだったら無視する