From 9a6462d00167cc208bb589cd2985a993fc154472 Mon Sep 17 00:00:00 2001 From: Kaity A Date: Tue, 4 Apr 2023 02:46:06 +1000 Subject: [PATCH 1/2] Requested limit to be fulfilled if possible --- .../api/endpoints/notes/global-timeline.ts | 20 +++++++++++++--- .../api/endpoints/notes/hybrid-timeline.ts | 20 +++++++++++++--- .../api/endpoints/notes/local-timeline.ts | 20 +++++++++++++--- .../server/api/endpoints/notes/mentions.ts | 21 +++++++++++++--- .../endpoints/notes/recommended-timeline.ts | 20 +++++++++++++--- .../src/server/api/endpoints/notes/renotes.ts | 18 ++++++++++++-- .../src/server/api/endpoints/notes/replies.ts | 18 ++++++++++++-- .../api/endpoints/notes/search-by-tag.ts | 19 ++++++++++++--- .../server/api/endpoints/notes/timeline.ts | 20 +++++++++++++--- .../api/endpoints/notes/user-list-timeline.ts | 24 ++++++++++++++++--- 10 files changed, 172 insertions(+), 28 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts index e4a38cffb1..bf4f0dcf29 100644 --- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts @@ -93,13 +93,27 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { if (user) { activeUsersChart.read(user); } }); + + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(timeline, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 3d6103da87..fcd1aa3b54 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -151,11 +151,25 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { activeUsersChart.read(user); }); - return await Notes.packMany(timeline, user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } + + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 22e5965fce..a1bc4a4eba 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -123,13 +123,27 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { if (user) { activeUsersChart.read(user); } }); - return await Notes.packMany(timeline, user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } + + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/mentions.ts b/packages/backend/src/server/api/endpoints/notes/mentions.ts index 3e5c4f18b2..9bd691c5f6 100644 --- a/packages/backend/src/server/api/endpoints/notes/mentions.ts +++ b/packages/backend/src/server/api/endpoints/notes/mentions.ts @@ -86,9 +86,24 @@ export default define(meta, paramDef, async (ps, user) => { query.setParameters(followingQuery.getParameters()); } - const mentions = await query.take(ps.limit).getMany(); - read(user.id, mentions); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(mentions, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + read(user.id, found); + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts index 6dacadec2a..1e637978ef 100644 --- a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts @@ -126,13 +126,27 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { if (user) { activeUsersChart.read(user); } }); - return await Notes.packMany(timeline, user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } + + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/renotes.ts b/packages/backend/src/server/api/endpoints/notes/renotes.ts index b09243e7e6..5ac0e47c16 100644 --- a/packages/backend/src/server/api/endpoints/notes/renotes.ts +++ b/packages/backend/src/server/api/endpoints/notes/renotes.ts @@ -74,7 +74,21 @@ export default define(meta, paramDef, async (ps, user) => { if (user) generateMutedUserQuery(query, user); if (user) generateBlockedUserQuery(query, user); - const renotes = await query.take(ps.limit).getMany(); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(renotes, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/replies.ts b/packages/backend/src/server/api/endpoints/notes/replies.ts index 0a8e909496..0cabc94008 100644 --- a/packages/backend/src/server/api/endpoints/notes/replies.ts +++ b/packages/backend/src/server/api/endpoints/notes/replies.ts @@ -58,7 +58,21 @@ export default define(meta, paramDef, async (ps, user) => { if (user) generateMutedUserQuery(query, user); if (user) generateBlockedUserQuery(query, user); - const timeline = await query.take(ps.limit).getMany(); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(timeline, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts index d8d0dbbf73..5e232e4d56 100644 --- a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts +++ b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts @@ -145,8 +145,21 @@ export default define(meta, paramDef, async (ps, me) => { } } - // Search notes - const notes = await query.take(ps.limit).getMany(); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, me)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(notes, me); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 083f41365a..0bb53fd4ab 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -143,11 +143,25 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { activeUsersChart.read(user); }); - return await Notes.packMany(timeline, user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } + + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index c1e5357222..bc75954c81 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -138,9 +138,27 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); + process.nextTick(() => { + if (user) { + activeUsersChart.read(user); + } + }); - activeUsersChart.read(user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. But if there's more nobody cares. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(timeline, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); From 3d93458d8b72f18985c2d169d06fa0696fab41b1 Mon Sep 17 00:00:00 2001 From: Kaity A Date: Tue, 4 Apr 2023 02:46:06 +1000 Subject: [PATCH 2/2] Requested limit to be fulfilled if possible --- .../api/endpoints/notes/global-timeline.ts | 20 +++++++++++++--- .../api/endpoints/notes/hybrid-timeline.ts | 20 +++++++++++++--- .../api/endpoints/notes/local-timeline.ts | 20 +++++++++++++--- .../server/api/endpoints/notes/mentions.ts | 21 +++++++++++++--- .../endpoints/notes/recommended-timeline.ts | 20 +++++++++++++--- .../src/server/api/endpoints/notes/renotes.ts | 18 ++++++++++++-- .../src/server/api/endpoints/notes/replies.ts | 18 ++++++++++++-- .../api/endpoints/notes/search-by-tag.ts | 19 ++++++++++++--- .../server/api/endpoints/notes/timeline.ts | 20 +++++++++++++--- .../api/endpoints/notes/user-list-timeline.ts | 24 ++++++++++++++++--- 10 files changed, 172 insertions(+), 28 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts index e4a38cffb1..584a6ce020 100644 --- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts @@ -93,13 +93,27 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { if (user) { activeUsersChart.read(user); } }); + + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(timeline, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 3d6103da87..78846861ad 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -151,11 +151,25 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { activeUsersChart.read(user); }); - return await Notes.packMany(timeline, user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } + + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 22e5965fce..fead877387 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -123,13 +123,27 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { if (user) { activeUsersChart.read(user); } }); - return await Notes.packMany(timeline, user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } + + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/mentions.ts b/packages/backend/src/server/api/endpoints/notes/mentions.ts index 3e5c4f18b2..7ffe83e5c9 100644 --- a/packages/backend/src/server/api/endpoints/notes/mentions.ts +++ b/packages/backend/src/server/api/endpoints/notes/mentions.ts @@ -86,9 +86,24 @@ export default define(meta, paramDef, async (ps, user) => { query.setParameters(followingQuery.getParameters()); } - const mentions = await query.take(ps.limit).getMany(); - read(user.id, mentions); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(mentions, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + read(user.id, found); + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts index 6dacadec2a..c73114de61 100644 --- a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts @@ -126,13 +126,27 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { if (user) { activeUsersChart.read(user); } }); - return await Notes.packMany(timeline, user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } + + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/renotes.ts b/packages/backend/src/server/api/endpoints/notes/renotes.ts index b09243e7e6..60d398088a 100644 --- a/packages/backend/src/server/api/endpoints/notes/renotes.ts +++ b/packages/backend/src/server/api/endpoints/notes/renotes.ts @@ -74,7 +74,21 @@ export default define(meta, paramDef, async (ps, user) => { if (user) generateMutedUserQuery(query, user); if (user) generateBlockedUserQuery(query, user); - const renotes = await query.take(ps.limit).getMany(); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(renotes, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/replies.ts b/packages/backend/src/server/api/endpoints/notes/replies.ts index 0a8e909496..94a752a64b 100644 --- a/packages/backend/src/server/api/endpoints/notes/replies.ts +++ b/packages/backend/src/server/api/endpoints/notes/replies.ts @@ -58,7 +58,21 @@ export default define(meta, paramDef, async (ps, user) => { if (user) generateMutedUserQuery(query, user); if (user) generateBlockedUserQuery(query, user); - const timeline = await query.take(ps.limit).getMany(); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(timeline, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts index d8d0dbbf73..f1cae78ba9 100644 --- a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts +++ b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts @@ -145,8 +145,21 @@ export default define(meta, paramDef, async (ps, me) => { } } - // Search notes - const notes = await query.take(ps.limit).getMany(); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, me)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(notes, me); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 083f41365a..e79f6b5898 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -143,11 +143,25 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); - process.nextTick(() => { activeUsersChart.read(user); }); - return await Notes.packMany(timeline, user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } + + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; }); diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index c1e5357222..1123a33ae7 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -138,9 +138,27 @@ export default define(meta, paramDef, async (ps, user) => { } //#endregion - const timeline = await query.take(ps.limit).getMany(); + process.nextTick(() => { + if (user) { + activeUsersChart.read(user); + } + }); - activeUsersChart.read(user); + // We fetch more than requested because some may be filtered out, and if there's less than + // requested, the pagination stops. + const found = []; + const take = Math.floor(ps.limit * 1.5); + let skip = 0; + while (found.length < ps.limit) { + const notes = await query.take(take).skip(skip).getMany(); + found.push(...await Notes.packMany(notes, user)) + skip += take; + if (notes.length < take) break; + } - return await Notes.packMany(timeline, user); + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; });