From 98f9e3e5c47b29bbc3a4c8eab9787ee1de90529d Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 28 May 2023 20:24:48 -0400 Subject: [PATCH 1/2] use cuid2 with timestamp --- .config/example.yml | 22 ++++------ .../native-utils/__test__/index.spec.mjs | 8 ++-- packages/backend/package.json | 1 + packages/backend/src/config/types.ts | 5 ++- packages/backend/src/misc/gen-id.ts | 42 +++++++++---------- pnpm-lock.yaml | 13 ++++++ 6 files changed, 53 insertions(+), 38 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 7d8ba32be6..a83e02eda3 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -85,20 +85,16 @@ redis: # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── -# You can select the ID generation method. -# You don't usually need to change this setting, but you can -# change it according to your preferences. +# No need to uncomment in most cases, but you may want to change +# these settings if you plan to run a large and/or distributed server. -# Available methods: -# aid ... Short, Millisecond accuracy -# meid ... Similar to ObjectID, Millisecond accuracy -# ulid ... Millisecond accuracy -# objectid ... This is left for backward compatibility - -# ONCE YOU HAVE STARTED THE INSTANCE, DO NOT CHANGE THE -# ID SETTINGS AFTER THAT! - -id: 'aid' +# cuid: { +# # Min 16, Max 24 +# length: 16, +# # Set this to a unique string across workers (e.g., machine's hostname) +# # if your workers are running in multiple hosts. +# fingerprint: "my-fingerperint", +# } # ┌─────────────────────┐ #───┘ Other configuration └───────────────────────────────────── diff --git a/packages/backend/native-utils/__test__/index.spec.mjs b/packages/backend/native-utils/__test__/index.spec.mjs index 0d41e012dd..7d68d6ac36 100644 --- a/packages/backend/native-utils/__test__/index.spec.mjs +++ b/packages/backend/native-utils/__test__/index.spec.mjs @@ -1,7 +1,9 @@ import test from "ava"; -import { sum } from "../index.js"; +import { convertId, IdConvertType } from "../built/index.js"; -test("sum from native", (t) => { - t.is(sum(1, 2), 3); +test("convert to mastodon id", (t) => { + t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219"); + t.is(convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId), "3954607381600562394"); + t.is(convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId), "3494513243013053824") }); diff --git a/packages/backend/package.json b/packages/backend/package.json index 96edb7f026..c0c5e24f90 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -32,6 +32,7 @@ "@koa/cors": "3.4.3", "@koa/multer": "3.0.0", "@koa/router": "9.0.1", + "@paralleldrive/cuid2": "2.2.0", "@peertube/http-signature": "1.7.0", "@redocly/openapi-core": "1.0.0-beta.120", "@sinonjs/fake-timers": "9.1.2", diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index 01a98f9f09..e01c67ade1 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -54,7 +54,10 @@ export type Source = { onlyQueueProcessor?: boolean; - id: string; + cuid?: { + length?: number; + fingerprint?: string; + }; outgoingAddressFamily?: "ipv4" | "ipv6" | "dual"; diff --git a/packages/backend/src/misc/gen-id.ts b/packages/backend/src/misc/gen-id.ts index b7cc0965a1..fb92dd808c 100644 --- a/packages/backend/src/misc/gen-id.ts +++ b/packages/backend/src/misc/gen-id.ts @@ -1,27 +1,27 @@ -import { ulid } from "ulid"; -import { genAid } from "./id/aid.js"; -import { genMeid } from "./id/meid.js"; -import { genMeidg } from "./id/meidg.js"; -import { genObjectId } from "./id/object-id.js"; +import { init, createId } from "@paralleldrive/cuid2"; import config from "@/config/index.js"; -const metohd = config.id.toLowerCase(); +const TIME2000 = 946684800000; +const TIMESTAMP_LENGTH = 8; +const length = + Math.min(Math.max(config.cuid?.length ?? 16, 16), 24) - TIMESTAMP_LENGTH; +const fingerprint = `${config.cuid?.fingerprint ?? ""}${createId()}`; + +const genCuid2 = init({ length, fingerprint }); + +/** + * The generated ID results in the form of `[8 chars timestamp] + [cuid2]`. + * The minimum and maximum lengths are 16 and 24, respectively. + * With the length of 16, namely 8 for cuid2, roughly 1427399 IDs are needed + * in the same millisecond to reach 50% chance of collision. + * + * Ref: https://github.com/paralleldrive/cuid2#parameterized-length + */ export function genId(date?: Date): string { - if (!date || date > new Date()) date = new Date(); + const now = (date ?? new Date()).getTime(); + const time = Math.max(now - TIME2000, 0); + const timestamp = time.toString(36).padStart(TIMESTAMP_LENGTH, "0"); - switch (metohd) { - case "aid": - return genAid(date); - case "meid": - return genMeid(date); - case "meidg": - return genMeidg(date); - case "ulid": - return ulid(date.getTime()); - case "objectid": - return genObjectId(date); - default: - throw new Error("unrecognized id generation method"); - } + return `${timestamp}${genCuid2()}`; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 493e9fc062..b1ec8dbd64 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,6 +104,9 @@ importers: '@koa/router': specifier: 9.0.1 version: 9.0.1 + '@paralleldrive/cuid2': + specifier: 2.2.0 + version: 2.2.0 '@peertube/http-signature': specifier: 1.7.0 version: 1.7.0 @@ -2277,6 +2280,10 @@ packages: hasBin: true dev: false + /@noble/hashes@1.3.0: + resolution: {integrity: sha512-ilHEACi9DwqJB0pw7kv+Apvh50jiiSyR/cQ3y4W7lOR5mhvn/50FLUfsnfJz0BDZtl/RR16kXvptiv6q1msYZg==} + dev: false + /@nodelib/fs.scandir@2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -2330,6 +2337,12 @@ packages: through: 2.3.4 dev: false + /@paralleldrive/cuid2@2.2.0: + resolution: {integrity: sha512-CVQDpPIUHrUGGLdrMGz1NmqZvqmsB2j2rCIQEu1EvxWjlFh4fhvEGmgR409cY20/67/WlJsggenq0no3p3kYsw==} + dependencies: + '@noble/hashes': 1.3.0 + dev: false + /@peertube/http-signature@1.7.0: resolution: {integrity: sha512-aGQIwo6/sWtyyqhVK4e1MtxYz4N1X8CNt6SOtCc+Wnczs5S5ONaLHDDR8LYaGn0MgOwvGgXyuZ5sJIfd7iyoUw==} engines: {node: '>=0.10'} From cf4427f4f094294034354b66b99d96f3626ab837 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 28 May 2023 23:49:55 -0400 Subject: [PATCH 2/2] fix example config format --- .config/example.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index a83e02eda3..f5248fb9a0 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -88,13 +88,14 @@ redis: # No need to uncomment in most cases, but you may want to change # these settings if you plan to run a large and/or distributed server. -# cuid: { +# cuid: # # Min 16, Max 24 -# length: 16, +# length: 16 +# # # Set this to a unique string across workers (e.g., machine's hostname) # # if your workers are running in multiple hosts. -# fingerprint: "my-fingerperint", -# } +# fingerprint: my-fingerprint + # ┌─────────────────────┐ #───┘ Other configuration └─────────────────────────────────────