From 79a7e6c232723319ae68faccc9359d86989c07de Mon Sep 17 00:00:00 2001 From: Natty Date: Mon, 1 Jan 2024 16:04:33 +0100 Subject: [PATCH] Switched ID generation to ULID --- .config/ci.yml | 18 ----------- .config/example.yml | 15 --------- packages/backend/package.json | 3 +- packages/backend/src/config/types.ts | 5 --- packages/backend/src/misc/gen-id.ts | 31 ++---------------- .../processors/db/export-custom-emojis.ts | 16 ++++------ pnpm-lock.yaml | 32 +++++++------------ 7 files changed, 22 insertions(+), 98 deletions(-) diff --git a/.config/ci.yml b/.config/ci.yml index 0f5be623bc..0fba833790 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -72,24 +72,6 @@ redis: # user: # pass: -# ┌───────────────┐ -#───┘ ID generation └─────────────────────────────────────────── - -# You can select the ID generation method. -# You don't usually need to change this setting, but you can -# change it according to your preferences. - -# Available methods: -# aid ... Short, Millisecond accuracy -# meid ... Similar to ObjectID, Millisecond accuracy -# ulid ... Millisecond accuracy -# objectid ... This is left for backward compatibility - -# ONCE YOU HAVE STARTED THE INSTANCE, DO NOT CHANGE THE -# ID SETTINGS AFTER THAT! - -id: 'aid' - # ┌─────────────────────┐ #───┘ Other configuration └───────────────────────────────────── diff --git a/.config/example.yml b/.config/example.yml index 16fa67142e..84c56ad7a7 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -83,21 +83,6 @@ redis: # bucket: default -# ┌───────────────┐ -#───┘ ID generation └─────────────────────────────────────────── - -# No need to uncomment in most cases, but you may want to change -# these settings if you plan to run a large and/or distributed server. - -# cuid: -# # Min 16, Max 24 -# length: 16 -# -# # Set this to a unique string across workers (e.g., machine's hostname) -# # if your workers are running in multiple hosts. -# fingerprint: my-fingerprint - - # ┌─────────────────────┐ #───┘ Other configuration └───────────────────────────────────── diff --git a/packages/backend/package.json b/packages/backend/package.json index face9d5e1b..912d765429 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -20,7 +20,6 @@ "@koa/cors": "3.4.3", "@koa/multer": "3.0.2", "@koa/router": "9.0.1", - "@paralleldrive/cuid2": "2.2.1", "@peertube/http-signature": "1.7.0", "@redocly/openapi-core": "1.0.0-beta.120", "@sinonjs/fake-timers": "9.1.2", @@ -110,7 +109,7 @@ "tmp": "0.2.1", "twemoji-parser": "14.0.0", "typeorm": "0.3.11", - "ulid": "2.3.0", + "ulidx": "2.2.1", "uuid": "9.0.0", "web-push": "3.6.1", "websocket": "1.0.34", diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index 38ca0a8f82..9c311c35fa 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -60,11 +60,6 @@ export type Source = { onlyQueueProcessor?: boolean; - cuid?: { - length?: number; - fingerprint?: string; - }; - outgoingAddressFamily?: "ipv4" | "ipv6" | "dual"; deliverJobConcurrency?: number; diff --git a/packages/backend/src/misc/gen-id.ts b/packages/backend/src/misc/gen-id.ts index 6ba6e1fc98..92c221ab27 100644 --- a/packages/backend/src/misc/gen-id.ts +++ b/packages/backend/src/misc/gen-id.ts @@ -1,32 +1,5 @@ -import config from "@/config/index.js"; -import {init} from "@paralleldrive/cuid2"; -import {randomInt} from "crypto"; +import {ulid} from "ulidx"; -const length = Math.min(Math.max(config.cuid?.length ?? 16, 16), 24); -const fingerprint = config.cuid?.fingerprint ?? ""; -const timestampLength = 8; - -const createId = init({ - random: () => { - return randomInt(0, 0xffff_ffff_ffff - 1) / 0xffff_ffff_ffff; - }, - length: Math.max(length - timestampLength, 8), - fingerprint, -}); - -/** - * The generated ID results in the form of `[8 chars timestamp] + [cuid2]`. - * The minimum and maximum lengths are 16 and 24, respectively. - * With the length of 16, namely 8 for cuid2, roughly 1427399 IDs are needed - * in the same millisecond to reach 50% chance of collision. - * - * Ref: https://github.com/paralleldrive/cuid2#parameterized-length - */ export function genId(date?: Date): string { - const time2000 = BigInt(946_684_800_000); - const time = BigInt((date ?? new Date()).getTime()) - time2000; - const time36 = time.toString(36).padStart(8, "0"); - const cuid = createId(); - - return `${time36}${cuid}`; + return ulid(date?.getTime()); } diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts index 7a19d0b60b..1fb9013a0e 100644 --- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -1,18 +1,16 @@ import type Bull from "bull"; import * as fs from "node:fs"; -import { ulid } from "ulid"; import mime from "mime-types"; import archiver from "archiver"; -import { queueLogger } from "../../logger.js"; -import { addFile } from "@/services/drive/add-file.js"; -import { format as dateFormat } from "date-fns"; -import { Users, Emojis } from "@/models/index.js"; -import {} from "@/queue/types.js"; -import { createTemp, createTempDir } from "@/misc/create-temp.js"; -import { downloadUrl } from "@/misc/download-url.js"; +import {queueLogger} from "../../logger.js"; +import {addFile} from "@/services/drive/add-file.js"; +import {format as dateFormat} from "date-fns"; +import {Emojis, Users} from "@/models/index.js"; +import {createTemp, createTempDir} from "@/misc/create-temp.js"; +import {downloadUrl} from "@/misc/download-url.js"; import config from "@/config/index.js"; -import { IsNull } from "typeorm"; +import {IsNull} from "typeorm"; const logger = queueLogger.createSubLogger("export-custom-emojis"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d80832b61b..3ee0b4d9b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -63,9 +63,6 @@ importers: '@koa/router': specifier: 9.0.1 version: 9.0.1 - '@paralleldrive/cuid2': - specifier: 2.2.1 - version: 2.2.1 '@peertube/http-signature': specifier: 1.7.0 version: 1.7.0 @@ -333,9 +330,9 @@ importers: typeorm: specifier: 0.3.11 version: 0.3.11(ioredis@5.3.2)(pg@8.11.0)(ts-node@10.9.1) - ulid: - specifier: 2.3.0 - version: 2.3.0 + ulidx: + specifier: 2.2.1 + version: 2.2.1 uuid: specifier: 9.0.0 version: 9.0.0 @@ -1123,11 +1120,6 @@ packages: dev: false optional: true - /@noble/hashes@1.3.1: - resolution: {integrity: sha512-EbqwksQwz9xDRGfDST86whPBgM65E0OH/pCgqW0GBVzO22bNE+NuIbeTb714+IfSjU3aRk47EUvXIb5bTsenKA==} - engines: {node: '>= 16'} - dev: false - /@nodelib/fs.scandir@2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1166,12 +1158,6 @@ packages: semver: 7.5.1 dev: false - /@paralleldrive/cuid2@2.2.1: - resolution: {integrity: sha512-GJhHYlMhyT2gWemDL7BGMWfTNhspJKkikLKh9wAy3z4GTTINvTYALkUd+eGQK7aLeVkVzPuSA0VCT3H5eEWbbw==} - dependencies: - '@noble/hashes': 1.3.1 - dev: false - /@peertube/http-signature@1.7.0: resolution: {integrity: sha512-aGQIwo6/sWtyyqhVK4e1MtxYz4N1X8CNt6SOtCc+Wnczs5S5ONaLHDDR8LYaGn0MgOwvGgXyuZ5sJIfd7iyoUw==} engines: {node: '>=0.10'} @@ -5187,6 +5173,10 @@ packages: engines: {node: '>=14.16'} dev: false + /layerr@2.0.1: + resolution: {integrity: sha512-z0730CwG/JO24evdORnyDkwG1Q7b7mF2Tp1qRQ0YvrMMARbt1DFG694SOv439Gm7hYKolyZyaB49YIrYIfZBdg==} + dev: false + /lazy-ass@1.6.0: resolution: {integrity: sha512-cc8oEVoctTvsFZ/Oje/kGnHbpWHYBe8IAJe4C0QNc3t8uM/0Y8+erSz/7Y1ALuXTEZTMvxXwO6YbX1ey3ujiZw==} engines: {node: '> 0.8'} @@ -7500,9 +7490,11 @@ packages: engines: {node: '>=14.17'} hasBin: true - /ulid@2.3.0: - resolution: {integrity: sha512-keqHubrlpvT6G2wH0OEfSW4mquYRcbe/J8NMmveoQOjUqmo+hXtO+ORCpWhdbZ7k72UtY61BL7haGxW6enBnjw==} - hasBin: true + /ulidx@2.2.1: + resolution: {integrity: sha512-DU9F5t1tihdafuNyW3fIrXUFHHiHxmwuQSGVGIbSpqkc93IH4P0dU8nPhk0gOW7ARxaFu4+P/9cxVwn6PdnIaQ==} + engines: {node: '>=16'} + dependencies: + layerr: 2.0.1 dev: false /undici@5.22.1: