Switched ID generation to ULID
This commit is contained in:
parent
5bed95e357
commit
79a7e6c232
|
@ -72,24 +72,6 @@ redis:
|
||||||
# user:
|
# user:
|
||||||
# pass:
|
# pass:
|
||||||
|
|
||||||
# ┌───────────────┐
|
|
||||||
#───┘ ID generation └───────────────────────────────────────────
|
|
||||||
|
|
||||||
# You can select the ID generation method.
|
|
||||||
# You don't usually need to change this setting, but you can
|
|
||||||
# change it according to your preferences.
|
|
||||||
|
|
||||||
# Available methods:
|
|
||||||
# aid ... Short, Millisecond accuracy
|
|
||||||
# meid ... Similar to ObjectID, Millisecond accuracy
|
|
||||||
# ulid ... Millisecond accuracy
|
|
||||||
# objectid ... This is left for backward compatibility
|
|
||||||
|
|
||||||
# ONCE YOU HAVE STARTED THE INSTANCE, DO NOT CHANGE THE
|
|
||||||
# ID SETTINGS AFTER THAT!
|
|
||||||
|
|
||||||
id: 'aid'
|
|
||||||
|
|
||||||
# ┌─────────────────────┐
|
# ┌─────────────────────┐
|
||||||
#───┘ Other configuration └─────────────────────────────────────
|
#───┘ Other configuration └─────────────────────────────────────
|
||||||
|
|
||||||
|
|
|
@ -83,21 +83,6 @@ redis:
|
||||||
# bucket: default
|
# bucket: default
|
||||||
|
|
||||||
|
|
||||||
# ┌───────────────┐
|
|
||||||
#───┘ ID generation └───────────────────────────────────────────
|
|
||||||
|
|
||||||
# No need to uncomment in most cases, but you may want to change
|
|
||||||
# these settings if you plan to run a large and/or distributed server.
|
|
||||||
|
|
||||||
# cuid:
|
|
||||||
# # Min 16, Max 24
|
|
||||||
# length: 16
|
|
||||||
#
|
|
||||||
# # Set this to a unique string across workers (e.g., machine's hostname)
|
|
||||||
# # if your workers are running in multiple hosts.
|
|
||||||
# fingerprint: my-fingerprint
|
|
||||||
|
|
||||||
|
|
||||||
# ┌─────────────────────┐
|
# ┌─────────────────────┐
|
||||||
#───┘ Other configuration └─────────────────────────────────────
|
#───┘ Other configuration └─────────────────────────────────────
|
||||||
|
|
||||||
|
|
|
@ -20,7 +20,6 @@
|
||||||
"@koa/cors": "3.4.3",
|
"@koa/cors": "3.4.3",
|
||||||
"@koa/multer": "3.0.2",
|
"@koa/multer": "3.0.2",
|
||||||
"@koa/router": "9.0.1",
|
"@koa/router": "9.0.1",
|
||||||
"@paralleldrive/cuid2": "2.2.1",
|
|
||||||
"@peertube/http-signature": "1.7.0",
|
"@peertube/http-signature": "1.7.0",
|
||||||
"@redocly/openapi-core": "1.0.0-beta.120",
|
"@redocly/openapi-core": "1.0.0-beta.120",
|
||||||
"@sinonjs/fake-timers": "9.1.2",
|
"@sinonjs/fake-timers": "9.1.2",
|
||||||
|
@ -110,7 +109,7 @@
|
||||||
"tmp": "0.2.1",
|
"tmp": "0.2.1",
|
||||||
"twemoji-parser": "14.0.0",
|
"twemoji-parser": "14.0.0",
|
||||||
"typeorm": "0.3.11",
|
"typeorm": "0.3.11",
|
||||||
"ulid": "2.3.0",
|
"ulidx": "2.2.1",
|
||||||
"uuid": "9.0.0",
|
"uuid": "9.0.0",
|
||||||
"web-push": "3.6.1",
|
"web-push": "3.6.1",
|
||||||
"websocket": "1.0.34",
|
"websocket": "1.0.34",
|
||||||
|
|
|
@ -60,11 +60,6 @@ export type Source = {
|
||||||
|
|
||||||
onlyQueueProcessor?: boolean;
|
onlyQueueProcessor?: boolean;
|
||||||
|
|
||||||
cuid?: {
|
|
||||||
length?: number;
|
|
||||||
fingerprint?: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
outgoingAddressFamily?: "ipv4" | "ipv6" | "dual";
|
outgoingAddressFamily?: "ipv4" | "ipv6" | "dual";
|
||||||
|
|
||||||
deliverJobConcurrency?: number;
|
deliverJobConcurrency?: number;
|
||||||
|
|
|
@ -1,32 +1,5 @@
|
||||||
import config from "@/config/index.js";
|
import {ulid} from "ulidx";
|
||||||
import {init} from "@paralleldrive/cuid2";
|
|
||||||
import {randomInt} from "crypto";
|
|
||||||
|
|
||||||
const length = Math.min(Math.max(config.cuid?.length ?? 16, 16), 24);
|
|
||||||
const fingerprint = config.cuid?.fingerprint ?? "";
|
|
||||||
const timestampLength = 8;
|
|
||||||
|
|
||||||
const createId = init({
|
|
||||||
random: () => {
|
|
||||||
return randomInt(0, 0xffff_ffff_ffff - 1) / 0xffff_ffff_ffff;
|
|
||||||
},
|
|
||||||
length: Math.max(length - timestampLength, 8),
|
|
||||||
fingerprint,
|
|
||||||
});
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The generated ID results in the form of `[8 chars timestamp] + [cuid2]`.
|
|
||||||
* The minimum and maximum lengths are 16 and 24, respectively.
|
|
||||||
* With the length of 16, namely 8 for cuid2, roughly 1427399 IDs are needed
|
|
||||||
* in the same millisecond to reach 50% chance of collision.
|
|
||||||
*
|
|
||||||
* Ref: https://github.com/paralleldrive/cuid2#parameterized-length
|
|
||||||
*/
|
|
||||||
export function genId(date?: Date): string {
|
export function genId(date?: Date): string {
|
||||||
const time2000 = BigInt(946_684_800_000);
|
return ulid(date?.getTime());
|
||||||
const time = BigInt((date ?? new Date()).getTime()) - time2000;
|
|
||||||
const time36 = time.toString(36).padStart(8, "0");
|
|
||||||
const cuid = createId();
|
|
||||||
|
|
||||||
return `${time36}${cuid}`;
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,18 +1,16 @@
|
||||||
import type Bull from "bull";
|
import type Bull from "bull";
|
||||||
import * as fs from "node:fs";
|
import * as fs from "node:fs";
|
||||||
|
|
||||||
import { ulid } from "ulid";
|
|
||||||
import mime from "mime-types";
|
import mime from "mime-types";
|
||||||
import archiver from "archiver";
|
import archiver from "archiver";
|
||||||
import { queueLogger } from "../../logger.js";
|
import {queueLogger} from "../../logger.js";
|
||||||
import { addFile } from "@/services/drive/add-file.js";
|
import {addFile} from "@/services/drive/add-file.js";
|
||||||
import { format as dateFormat } from "date-fns";
|
import {format as dateFormat} from "date-fns";
|
||||||
import { Users, Emojis } from "@/models/index.js";
|
import {Emojis, Users} from "@/models/index.js";
|
||||||
import {} from "@/queue/types.js";
|
import {createTemp, createTempDir} from "@/misc/create-temp.js";
|
||||||
import { createTemp, createTempDir } from "@/misc/create-temp.js";
|
import {downloadUrl} from "@/misc/download-url.js";
|
||||||
import { downloadUrl } from "@/misc/download-url.js";
|
|
||||||
import config from "@/config/index.js";
|
import config from "@/config/index.js";
|
||||||
import { IsNull } from "typeorm";
|
import {IsNull} from "typeorm";
|
||||||
|
|
||||||
const logger = queueLogger.createSubLogger("export-custom-emojis");
|
const logger = queueLogger.createSubLogger("export-custom-emojis");
|
||||||
|
|
||||||
|
|
|
@ -63,9 +63,6 @@ importers:
|
||||||
'@koa/router':
|
'@koa/router':
|
||||||
specifier: 9.0.1
|
specifier: 9.0.1
|
||||||
version: 9.0.1
|
version: 9.0.1
|
||||||
'@paralleldrive/cuid2':
|
|
||||||
specifier: 2.2.1
|
|
||||||
version: 2.2.1
|
|
||||||
'@peertube/http-signature':
|
'@peertube/http-signature':
|
||||||
specifier: 1.7.0
|
specifier: 1.7.0
|
||||||
version: 1.7.0
|
version: 1.7.0
|
||||||
|
@ -333,9 +330,9 @@ importers:
|
||||||
typeorm:
|
typeorm:
|
||||||
specifier: 0.3.11
|
specifier: 0.3.11
|
||||||
version: 0.3.11(ioredis@5.3.2)(pg@8.11.0)(ts-node@10.9.1)
|
version: 0.3.11(ioredis@5.3.2)(pg@8.11.0)(ts-node@10.9.1)
|
||||||
ulid:
|
ulidx:
|
||||||
specifier: 2.3.0
|
specifier: 2.2.1
|
||||||
version: 2.3.0
|
version: 2.2.1
|
||||||
uuid:
|
uuid:
|
||||||
specifier: 9.0.0
|
specifier: 9.0.0
|
||||||
version: 9.0.0
|
version: 9.0.0
|
||||||
|
@ -1123,11 +1120,6 @@ packages:
|
||||||
dev: false
|
dev: false
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
/@noble/hashes@1.3.1:
|
|
||||||
resolution: {integrity: sha512-EbqwksQwz9xDRGfDST86whPBgM65E0OH/pCgqW0GBVzO22bNE+NuIbeTb714+IfSjU3aRk47EUvXIb5bTsenKA==}
|
|
||||||
engines: {node: '>= 16'}
|
|
||||||
dev: false
|
|
||||||
|
|
||||||
/@nodelib/fs.scandir@2.1.5:
|
/@nodelib/fs.scandir@2.1.5:
|
||||||
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
|
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
|
||||||
engines: {node: '>= 8'}
|
engines: {node: '>= 8'}
|
||||||
|
@ -1166,12 +1158,6 @@ packages:
|
||||||
semver: 7.5.1
|
semver: 7.5.1
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
/@paralleldrive/cuid2@2.2.1:
|
|
||||||
resolution: {integrity: sha512-GJhHYlMhyT2gWemDL7BGMWfTNhspJKkikLKh9wAy3z4GTTINvTYALkUd+eGQK7aLeVkVzPuSA0VCT3H5eEWbbw==}
|
|
||||||
dependencies:
|
|
||||||
'@noble/hashes': 1.3.1
|
|
||||||
dev: false
|
|
||||||
|
|
||||||
/@peertube/http-signature@1.7.0:
|
/@peertube/http-signature@1.7.0:
|
||||||
resolution: {integrity: sha512-aGQIwo6/sWtyyqhVK4e1MtxYz4N1X8CNt6SOtCc+Wnczs5S5ONaLHDDR8LYaGn0MgOwvGgXyuZ5sJIfd7iyoUw==}
|
resolution: {integrity: sha512-aGQIwo6/sWtyyqhVK4e1MtxYz4N1X8CNt6SOtCc+Wnczs5S5ONaLHDDR8LYaGn0MgOwvGgXyuZ5sJIfd7iyoUw==}
|
||||||
engines: {node: '>=0.10'}
|
engines: {node: '>=0.10'}
|
||||||
|
@ -5187,6 +5173,10 @@ packages:
|
||||||
engines: {node: '>=14.16'}
|
engines: {node: '>=14.16'}
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
|
/layerr@2.0.1:
|
||||||
|
resolution: {integrity: sha512-z0730CwG/JO24evdORnyDkwG1Q7b7mF2Tp1qRQ0YvrMMARbt1DFG694SOv439Gm7hYKolyZyaB49YIrYIfZBdg==}
|
||||||
|
dev: false
|
||||||
|
|
||||||
/lazy-ass@1.6.0:
|
/lazy-ass@1.6.0:
|
||||||
resolution: {integrity: sha512-cc8oEVoctTvsFZ/Oje/kGnHbpWHYBe8IAJe4C0QNc3t8uM/0Y8+erSz/7Y1ALuXTEZTMvxXwO6YbX1ey3ujiZw==}
|
resolution: {integrity: sha512-cc8oEVoctTvsFZ/Oje/kGnHbpWHYBe8IAJe4C0QNc3t8uM/0Y8+erSz/7Y1ALuXTEZTMvxXwO6YbX1ey3ujiZw==}
|
||||||
engines: {node: '> 0.8'}
|
engines: {node: '> 0.8'}
|
||||||
|
@ -7500,9 +7490,11 @@ packages:
|
||||||
engines: {node: '>=14.17'}
|
engines: {node: '>=14.17'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
/ulid@2.3.0:
|
/ulidx@2.2.1:
|
||||||
resolution: {integrity: sha512-keqHubrlpvT6G2wH0OEfSW4mquYRcbe/J8NMmveoQOjUqmo+hXtO+ORCpWhdbZ7k72UtY61BL7haGxW6enBnjw==}
|
resolution: {integrity: sha512-DU9F5t1tihdafuNyW3fIrXUFHHiHxmwuQSGVGIbSpqkc93IH4P0dU8nPhk0gOW7ARxaFu4+P/9cxVwn6PdnIaQ==}
|
||||||
hasBin: true
|
engines: {node: '>=16'}
|
||||||
|
dependencies:
|
||||||
|
layerr: 2.0.1
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
/undici@5.22.1:
|
/undici@5.22.1:
|
||||||
|
|
Loading…
Reference in New Issue