Merge pull request 'Require timeline endpoints to return the requested limit of notes.' (#9805) from supakaity/hajkey:hk/notes-pagination into develop

Reviewed-on: https://codeberg.org/calckey/calckey/pulls/9805

packages/backend/src/server/api/endpoints/notes/global-timeline.ts

@@ -93,13 +93,27 @@ export default define(meta, paramDef, async (ps, user) => {
 	}
 	//#endregion
 
-	const timeline = await query.take(ps.limit).getMany();
-
 	process.nextTick(() => {
 		if (user) {
 			activeUsersChart.read(user);
 		}
 	});
+	
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(timeline, user);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts

@@ -151,11 +151,25 @@ export default define(meta, paramDef, async (ps, user) => {
 	}
 	//#endregion
 
-	const timeline = await query.take(ps.limit).getMany();
-
 	process.nextTick(() => {
 		activeUsersChart.read(user);
 	});
 
-	return await Notes.packMany(timeline, user);
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
+
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/local-timeline.ts

@@ -123,13 +123,27 @@ export default define(meta, paramDef, async (ps, user) => {
 	}
 	//#endregion
 
-	const timeline = await query.take(ps.limit).getMany();
-
 	process.nextTick(() => {
 		if (user) {
 			activeUsersChart.read(user);
 		}
 	});
 
-	return await Notes.packMany(timeline, user);
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
+
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/mentions.ts

@@ -86,9 +86,24 @@ export default define(meta, paramDef, async (ps, user) => {
 		query.setParameters(followingQuery.getParameters());
 	}
 
-	const mentions = await query.take(ps.limit).getMany();
 
-	read(user.id, mentions);
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(mentions, user);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	read(user.id, found);
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts

@@ -126,13 +126,27 @@ export default define(meta, paramDef, async (ps, user) => {
 	}
 	//#endregion
 
-	const timeline = await query.take(ps.limit).getMany();
-
 	process.nextTick(() => {
 		if (user) {
 			activeUsersChart.read(user);
 		}
 	});
 
-	return await Notes.packMany(timeline, user);
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
+
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/renotes.ts

@@ -74,7 +74,21 @@ export default define(meta, paramDef, async (ps, user) => {
 	if (user) generateMutedUserQuery(query, user);
 	if (user) generateBlockedUserQuery(query, user);
 
-	const renotes = await query.take(ps.limit).getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(renotes, user);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/replies.ts

@@ -58,7 +58,21 @@ export default define(meta, paramDef, async (ps, user) => {
 	if (user) generateMutedUserQuery(query, user);
 	if (user) generateBlockedUserQuery(query, user);
 
-	const timeline = await query.take(ps.limit).getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(timeline, user);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/search-by-tag.ts

@@ -145,8 +145,21 @@ export default define(meta, paramDef, async (ps, me) => {
 		}
 	}
 
-	// Search notes
-	const notes = await query.take(ps.limit).getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, me))
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(notes, me);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/timeline.ts

@@ -143,11 +143,25 @@ export default define(meta, paramDef, async (ps, user) => {
 	}
 	//#endregion
 
-	const timeline = await query.take(ps.limit).getMany();
-
 	process.nextTick(() => {
 		activeUsersChart.read(user);
 	});
 
-	return await Notes.packMany(timeline, user);
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
+
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts

@@ -138,9 +138,27 @@ export default define(meta, paramDef, async (ps, user) => {
 	}
 	//#endregion
 
-	const timeline = await query.take(ps.limit).getMany();
+	process.nextTick(() => {
+		if (user) {
+			activeUsersChart.read(user);
+		}
+	});
 
-	activeUsersChart.read(user);
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...await Notes.packMany(notes, user))
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(timeline, user);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });