From 236d72685dfec013135fc1450b890e33ec377de1 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 10 Apr 2019 00:59:41 +0900 Subject: [PATCH] More puny --- src/models/repositories/drive-file.ts | 3 ++- src/queue/processors/inbox.ts | 26 ++++++++++++------- src/remote/activitypub/models/note.ts | 7 +++-- src/remote/activitypub/models/person.ts | 7 +++-- src/remote/activitypub/request.ts | 4 +-- src/remote/resolve-user.ts | 19 +++++--------- src/server/api/common/get-host-lower.ts | 6 ----- src/server/api/endpoints/admin/emoji/list.ts | 3 ++- .../admin/federation/update-instance.ts | 5 ++-- .../api/endpoints/federation/show-instance.ts | 3 ++- src/server/api/endpoints/users/followers.ts | 3 ++- src/server/api/endpoints/users/following.ts | 3 ++- src/server/api/index.ts | 5 +--- src/services/chart/charts/classes/instance.ts | 13 +++++----- 14 files changed, 52 insertions(+), 55 deletions(-) delete mode 100644 src/server/api/common/get-host-lower.ts diff --git a/src/models/repositories/drive-file.ts b/src/models/repositories/drive-file.ts index fe0ca72bfb..817677fa3b 100644 --- a/src/models/repositories/drive-file.ts +++ b/src/models/repositories/drive-file.ts @@ -3,6 +3,7 @@ import { DriveFile } from '../entities/drive-file'; import { Users, DriveFolders } from '..'; import rap from '@prezzemolo/rap'; import { User } from '../entities/user'; +import { toPuny } from '../../misc/convert-host'; @EntityRepository(DriveFile) export class DriveFileRepository extends Repository { @@ -39,7 +40,7 @@ export class DriveFileRepository extends Repository { public async clacDriveUsageOfHost(host: string): Promise { const { sum } = await this .createQueryBuilder('file') - .where('file.userHost = :host', { host: host }) + .where('file.userHost = :host', { host: toPuny(host) }) .select('SUM(file.size)', 'sum') .getRawOne(); diff --git a/src/queue/processors/inbox.ts b/src/queue/processors/inbox.ts index 16badabcf7..481bcbb1c4 100644 --- a/src/queue/processors/inbox.ts +++ b/src/queue/processors/inbox.ts @@ -4,7 +4,6 @@ import parseAcct from '../../misc/acct/parse'; import { IRemoteUser } from '../../models/entities/user'; import perform from '../../remote/activitypub/perform'; import { resolvePerson, updatePerson } from '../../remote/activitypub/models/person'; -import { toUnicode } from 'punycode'; import { URL } from 'url'; import { publishApLogStream } from '../../services/stream'; import Logger from '../../services/logger'; @@ -13,6 +12,7 @@ import { Instances, Users, UserPublickeys } from '../../models'; import { instanceChart } from '../../services/chart'; import { UserPublickey } from '../../models/entities/user-publickey'; import fetchMeta from '../../misc/fetch-meta'; +import { toPuny } from '../../misc/convert-host'; const logger = new Logger('inbox'); @@ -33,7 +33,10 @@ export default async (job: Bull.Job): Promise => { let key: UserPublickey; if (keyIdLower.startsWith('acct:')) { - const { username, host } = parseAcct(keyIdLower.slice('acct:'.length)); + const acct = parseAcct(keyIdLower.slice('acct:'.length)); + const host = toPuny(acct.host); + const username = toPuny(acct.username); + if (host === null) { logger.warn(`request was made by local user: @${username}`); return; @@ -50,19 +53,22 @@ export default async (job: Bull.Job): Promise => { // ブロックしてたら中断 // TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく const meta = await fetchMeta(); - if (meta.blockedHosts.includes(host.toLowerCase())) { + if (meta.blockedHosts.includes(host)) { logger.info(`Blocked request: ${host}`); return; } - user = await Users.findOne({ usernameLower: username, host: host.toLowerCase() }) as IRemoteUser; + user = await Users.findOne({ + usernameLower: username.toLowerCase(), + host: host + }) as IRemoteUser; key = await UserPublickeys.findOne({ userId: user.id }); } else { // アクティビティ内のホストの検証 - const host = toUnicode(new URL(signature.keyId).hostname.toLowerCase()); + const host = toPuny(new URL(signature.keyId).hostname); try { ValidateActivity(activity, host); } catch (e) { @@ -73,7 +79,7 @@ export default async (job: Bull.Job): Promise => { // ブロックしてたら中断 // TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく const meta = await fetchMeta(); - if (meta.blockedHosts.includes(host.toLowerCase())) { + if (meta.blockedHosts.includes(host)) { logger.info(`Blocked request: ${host}`); return; } @@ -145,7 +151,7 @@ export default async (job: Bull.Job): Promise => { function ValidateActivity(activity: any, host: string) { // id (if exists) if (typeof activity.id === 'string') { - const uriHost = toUnicode(new URL(activity.id).hostname.toLowerCase()); + const uriHost = toPuny(new URL(activity.id).hostname); if (host !== uriHost) { const diag = activity.signature ? '. Has LD-Signature. Forwarded?' : ''; throw new Error(`activity.id(${activity.id}) has different host(${host})${diag}`); @@ -154,7 +160,7 @@ function ValidateActivity(activity: any, host: string) { // actor (if exists) if (typeof activity.actor === 'string') { - const uriHost = toUnicode(new URL(activity.actor).hostname.toLowerCase()); + const uriHost = toPuny(new URL(activity.actor).hostname); if (host !== uriHost) throw new Error('activity.actor has different host'); } @@ -162,13 +168,13 @@ function ValidateActivity(activity: any, host: string) { if (activity.type === 'Create' && activity.object) { // object.id (if exists) if (typeof activity.object.id === 'string') { - const uriHost = toUnicode(new URL(activity.object.id).hostname.toLowerCase()); + const uriHost = toPuny(new URL(activity.object.id).hostname); if (host !== uriHost) throw new Error('activity.object.id has different host'); } // object.attributedTo (if exists) if (typeof activity.object.attributedTo === 'string') { - const uriHost = toUnicode(new URL(activity.object.attributedTo).hostname.toLowerCase()); + const uriHost = toPuny(new URL(activity.object.attributedTo).hostname); if (host !== uriHost) throw new Error('activity.object.attributedTo has different host'); } } diff --git a/src/remote/activitypub/models/note.ts b/src/remote/activitypub/models/note.ts index cd587c51cf..07991d4eb2 100644 --- a/src/remote/activitypub/models/note.ts +++ b/src/remote/activitypub/models/note.ts @@ -8,14 +8,13 @@ import { resolveImage } from './image'; import { IRemoteUser, User } from '../../../models/entities/user'; import { fromHtml } from '../../../mfm/fromHtml'; import { ITag, extractHashtags } from './tag'; -import { toUnicode } from 'punycode'; import { unique, concat, difference } from '../../../prelude/array'; import { extractPollFromQuestion } from './question'; import vote from '../../../services/note/polls/vote'; import { apLogger } from '../logger'; import { DriveFile } from '../../../models/entities/drive-file'; import { deliverQuestionUpdate } from '../../../services/note/polls/update'; -import { extractDbHost } from '../../../misc/convert-host'; +import { extractDbHost, toPuny } from '../../../misc/convert-host'; import { Notes, Emojis, Polls } from '../../../models'; import { Note } from '../../../models/entities/note'; import { IObject, INote } from '../type'; @@ -246,8 +245,8 @@ export async function resolveNote(value: string | IObject, resolver?: Resolver): return await createNote(uri, resolver); } -export async function extractEmojis(tags: ITag[], host_: string) { - const host = toUnicode(host_.toLowerCase()); +export async function extractEmojis(tags: ITag[], host: string) { + host = toPuny(host); if (!tags) return []; diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts index 2362455e02..715bae40e8 100644 --- a/src/remote/activitypub/models/person.ts +++ b/src/remote/activitypub/models/person.ts @@ -1,5 +1,4 @@ import * as promiseLimit from 'promise-limit'; -import { toUnicode } from 'punycode'; import config from '../../../config'; import Resolver from '../resolver'; @@ -33,7 +32,7 @@ const logger = apLogger; * @param uri Fetch target URI */ function validatePerson(x: any, uri: string) { - const expectHost = toUnicode(new URL(uri).hostname.toLowerCase()); + const expectHost = toPuny(new URL(uri).hostname); if (x == null) { return new Error('invalid person: object is null'); @@ -63,7 +62,7 @@ function validatePerson(x: any, uri: string) { return new Error('invalid person: id is not a string'); } - const idHost = toUnicode(new URL(x.id).hostname.toLowerCase()); + const idHost = toPuny(new URL(x.id).hostname); if (idHost !== expectHost) { return new Error('invalid person: id has different host'); } @@ -72,7 +71,7 @@ function validatePerson(x: any, uri: string) { return new Error('invalid person: publicKey.id is not a string'); } - const publicKeyIdHost = toUnicode(new URL(x.publicKey.id).hostname.toLowerCase()); + const publicKeyIdHost = toPuny(new URL(x.publicKey.id).hostname); if (publicKeyIdHost !== expectHost) { return new Error('invalid person: publicKey.id has different host'); } diff --git a/src/remote/activitypub/request.ts b/src/remote/activitypub/request.ts index c50d05e2a6..8aca5e8102 100644 --- a/src/remote/activitypub/request.ts +++ b/src/remote/activitypub/request.ts @@ -4,7 +4,6 @@ import { URL } from 'url'; import * as crypto from 'crypto'; import { lookup, IRunOptions } from 'lookup-dns-cache'; import * as promiseAny from 'promise-any'; -import { toUnicode } from 'punycode'; import config from '../../config'; import { ILocalUser } from '../../models/entities/user'; @@ -12,6 +11,7 @@ import { publishApLogStream } from '../../services/stream'; import { apLogger } from './logger'; import { UserKeypairs } from '../../models'; import fetchMeta from '../../misc/fetch-meta'; +import { toPuny } from '../../misc/convert-host'; export const logger = apLogger.createSubLogger('deliver'); @@ -25,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => { // ブロックしてたら中断 // TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく const meta = await fetchMeta(); - if (meta.blockedHosts.includes(toUnicode(host))) return; + if (meta.blockedHosts.includes(toPuny(host))) return; const data = JSON.stringify(object); diff --git a/src/remote/resolve-user.ts b/src/remote/resolve-user.ts index a10d3c2d84..e6a11bc0da 100644 --- a/src/remote/resolve-user.ts +++ b/src/remote/resolve-user.ts @@ -1,4 +1,3 @@ -import { toUnicode, toASCII } from 'punycode'; import webFinger from './webfinger'; import config from '../config'; import { createPerson, updatePerson } from './activitypub/models/person'; @@ -7,31 +6,27 @@ import { remoteLogger } from './logger'; import chalk from 'chalk'; import { User, IRemoteUser } from '../models/entities/user'; import { Users } from '../models'; +import { toPuny } from '../misc/convert-host'; const logger = remoteLogger.createSubLogger('resolve-user'); -export async function resolveUser(username: string, _host: string, option?: any, resync = false): Promise { +export async function resolveUser(username: string, host: string, option?: any, resync = false): Promise { const usernameLower = username.toLowerCase(); + host = toPuny(host); - if (_host == null) { + if (host == null) { logger.info(`return local user: ${usernameLower}`); return await Users.findOne({ usernameLower, host: null }); } - const configHostAscii = toASCII(config.host).toLowerCase(); - const configHost = toUnicode(configHostAscii); - - const hostAscii = toASCII(_host).toLowerCase(); - const host = toUnicode(hostAscii); - - if (configHost == host) { + if (config.host == host) { logger.info(`return local user: ${usernameLower}`); return await Users.findOne({ usernameLower, host: null }); } const user = await Users.findOne({ usernameLower, host }, option); - const acctLower = `${usernameLower}@${hostAscii}`; + const acctLower = `${usernameLower}@${host}`; if (user == null) { const self = await resolveSelf(acctLower); @@ -51,7 +46,7 @@ export async function resolveUser(username: string, _host: string, option?: any, // validate uri const uri = new URL(self.href); - if (uri.hostname !== hostAscii) { + if (uri.hostname !== host) { throw new Error(`Invalied uri`); } diff --git a/src/server/api/common/get-host-lower.ts b/src/server/api/common/get-host-lower.ts deleted file mode 100644 index 26ddf6c6d0..0000000000 --- a/src/server/api/common/get-host-lower.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { toUnicode } from 'punycode'; - -export default (host: string) => { - if (host == null) return null; - return toUnicode(host).toLowerCase(); -}; diff --git a/src/server/api/endpoints/admin/emoji/list.ts b/src/server/api/endpoints/admin/emoji/list.ts index 07174723b9..26385d4e23 100644 --- a/src/server/api/endpoints/admin/emoji/list.ts +++ b/src/server/api/endpoints/admin/emoji/list.ts @@ -1,6 +1,7 @@ import $ from 'cafy'; import define from '../../../define'; import { Emojis } from '../../../../../models'; +import { toPuny } from '../../../../../misc/convert-host'; export const meta = { desc: { @@ -22,7 +23,7 @@ export const meta = { export default define(meta, async (ps) => { const emojis = await Emojis.find({ - host: ps.host + host: toPuny(ps.host) }); return emojis.map(e => ({ diff --git a/src/server/api/endpoints/admin/federation/update-instance.ts b/src/server/api/endpoints/admin/federation/update-instance.ts index d1abe95a5b..90ab7a3ec5 100644 --- a/src/server/api/endpoints/admin/federation/update-instance.ts +++ b/src/server/api/endpoints/admin/federation/update-instance.ts @@ -1,6 +1,7 @@ import $ from 'cafy'; import define from '../../../define'; import { Instances } from '../../../../../models'; +import { toPuny } from '../../../../../misc/convert-host'; export const meta = { tags: ['admin'], @@ -20,13 +21,13 @@ export const meta = { }; export default define(meta, async (ps, me) => { - const instance = await Instances.findOne({ host: ps.host }); + const instance = await Instances.findOne({ host: toPuny(ps.host) }); if (instance == null) { throw new Error('instance not found'); } - Instances.update({ host: ps.host }, { + Instances.update({ host: toPuny(ps.host) }, { isMarkedAsClosed: ps.isClosed }); }); diff --git a/src/server/api/endpoints/federation/show-instance.ts b/src/server/api/endpoints/federation/show-instance.ts index 875afa05b2..9afcf28a7c 100644 --- a/src/server/api/endpoints/federation/show-instance.ts +++ b/src/server/api/endpoints/federation/show-instance.ts @@ -1,6 +1,7 @@ import $ from 'cafy'; import define from '../../define'; import { Instances } from '../../../../models'; +import { toPuny } from '../../../../misc/convert-host'; export const meta = { tags: ['federation'], @@ -16,7 +17,7 @@ export const meta = { export default define(meta, async (ps, me) => { const instance = await Instances - .findOne({ host: ps.host }); + .findOne({ host: toPuny(ps.host) }); return instance; }); diff --git a/src/server/api/endpoints/users/followers.ts b/src/server/api/endpoints/users/followers.ts index 51b007ddaa..64d63e2d03 100644 --- a/src/server/api/endpoints/users/followers.ts +++ b/src/server/api/endpoints/users/followers.ts @@ -4,6 +4,7 @@ import define from '../../define'; import { ApiError } from '../../error'; import { Users, Followings } from '../../../../models'; import { makePaginationQuery } from '../../common/make-pagination-query'; +import { toPuny } from '../../../../misc/convert-host'; export const meta = { desc: { @@ -65,7 +66,7 @@ export const meta = { export default define(meta, async (ps, me) => { const user = await Users.findOne(ps.userId != null ? { id: ps.userId } - : { usernameLower: ps.username.toLowerCase(), host: ps.host }); + : { usernameLower: ps.username.toLowerCase(), host: toPuny(ps.host) }); if (user == null) { throw new ApiError(meta.errors.noSuchUser); diff --git a/src/server/api/endpoints/users/following.ts b/src/server/api/endpoints/users/following.ts index 46550f0f77..0e28001680 100644 --- a/src/server/api/endpoints/users/following.ts +++ b/src/server/api/endpoints/users/following.ts @@ -4,6 +4,7 @@ import define from '../../define'; import { ApiError } from '../../error'; import { Users, Followings } from '../../../../models'; import { makePaginationQuery } from '../../common/make-pagination-query'; +import { toPuny } from '../../../../misc/convert-host'; export const meta = { desc: { @@ -65,7 +66,7 @@ export const meta = { export default define(meta, async (ps, me) => { const user = await Users.findOne(ps.userId != null ? { id: ps.userId } - : { usernameLower: ps.username.toLowerCase(), host: ps.host }); + : { usernameLower: ps.username.toLowerCase(), host: toPuny(ps.host) }); if (user == null) { throw new ApiError(meta.errors.noSuchUser); diff --git a/src/server/api/index.ts b/src/server/api/index.ts index 7858efd927..8c2b97775f 100644 --- a/src/server/api/index.ts +++ b/src/server/api/index.ts @@ -15,7 +15,6 @@ import signin from './private/signin'; import discord from './service/discord'; import github from './service/github'; import twitter from './service/twitter'; -import { toASCII } from 'punycode'; import { Instances } from '../../models'; // Init app @@ -71,9 +70,7 @@ router.get('/v1/instance/peers', async ctx => { select: ['host'] }); - const punyCodes = instances.map(instance => toASCII(instance.host)); - - ctx.body = punyCodes; + ctx.body = instances.map(instance => instance.host); }); // Return 404 for unknown API diff --git a/src/services/chart/charts/classes/instance.ts b/src/services/chart/charts/classes/instance.ts index 55db534573..f3d341f383 100644 --- a/src/services/chart/charts/classes/instance.ts +++ b/src/services/chart/charts/classes/instance.ts @@ -5,6 +5,7 @@ import { DriveFiles, Followings, Users, Notes } from '../../../../models'; import { DriveFile } from '../../../../models/entities/drive-file'; import { name, schema } from '../schemas/instance'; import { Note } from '../../../../models/entities/note'; +import { toPuny } from '../../../../misc/convert-host'; type InstanceLog = SchemaType; @@ -79,7 +80,7 @@ export default class InstanceChart extends Chart { requests: { received: 1 } - }, host); + }, toPuny(host)); } @autobind @@ -94,7 +95,7 @@ export default class InstanceChart extends Chart { await this.inc({ requests: update - }, host); + }, toPuny(host)); } @autobind @@ -104,7 +105,7 @@ export default class InstanceChart extends Chart { total: 1, inc: 1 } - }, host); + }, toPuny(host)); } @autobind @@ -126,7 +127,7 @@ export default class InstanceChart extends Chart { dec: isAdditional ? 0 : 1, diffs: diffs } - }, host); + }, toPuny(host)); } @autobind @@ -137,7 +138,7 @@ export default class InstanceChart extends Chart { inc: isAdditional ? 1 : 0, dec: isAdditional ? 0 : 1, } - }, host); + }, toPuny(host)); } @autobind @@ -148,7 +149,7 @@ export default class InstanceChart extends Chart { inc: isAdditional ? 1 : 0, dec: isAdditional ? 0 : 1, } - }, host); + }, toPuny(host)); } @autobind