Resolve #7149

src/misc/normalize-for-search.ts

@@ -0,0 +1,6 @@
+export function normalizeForSearch(tag: string): string {
+	// ref.
+	// - https://analytics-note.xyz/programming/unicode-normalization-forms/
+	// - https://maku77.github.io/js/string/normalize.html
+	return tag.normalize('NFKC').toLowerCase();
+}

src/remote/activitypub/models/person.ts

@@ -27,6 +27,7 @@ import { getConnection } from 'typeorm';
 import { ensure } from '../../../prelude/ensure';
 import { toArray } from '../../../prelude/array';
 import { fetchInstanceMetadata } from '../../../services/fetch-instance-metadata';
+import { normalizeForSearch } from '../../../misc/normalize-for-search';
 
 const logger = apLogger;
 
@@ -134,7 +135,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
 
 	const { fields } = analyzeAttachments(person.attachment || []);
 
-	const tags = extractApHashtags(person.tag).map(tag => tag.toLowerCase()).splice(0, 32);
+	const tags = extractApHashtags(person.tag).map(tag => normalizeForSearch(tag)).splice(0, 32);
 
 	const isBot = object.type === 'Service';
 
@@ -323,7 +324,7 @@ export async function updatePerson(uri: string, resolver?: Resolver | null, hint
 
 	const { fields } = analyzeAttachments(person.attachment || []);
 
-	const tags = extractApHashtags(person.tag).map(tag => tag.toLowerCase()).splice(0, 32);
+	const tags = extractApHashtags(person.tag).map(tag => normalizeForSearch(tag)).splice(0, 32);
 
 	const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/);
 

src/server/api/endpoints/hashtags/show.ts

@@ -2,6 +2,7 @@ import $ from 'cafy';
 import define from '../../define';
 import { ApiError } from '../../error';
 import { Hashtags } from '../../../../models';
+import { normalizeForSearch } from '../../../../misc/normalize-for-search';
 
 export const meta = {
 	desc: {
@@ -38,7 +39,7 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	const hashtag = await Hashtags.findOne({ name: ps.tag.toLowerCase() });
+	const hashtag = await Hashtags.findOne({ name: normalizeForSearch(ps.tag) });
 	if (hashtag == null) {
 		throw new ApiError(meta.errors.noSuchHashtag);
 	}

src/server/api/endpoints/hashtags/trend.ts

@@ -4,6 +4,7 @@ import { fetchMeta } from '../../../../misc/fetch-meta';
 import { Notes } from '../../../../models';
 import { Note } from '../../../../models/entities/note';
 import { safeForSql } from '../../../../misc/safe-for-sql';
+import { normalizeForSearch } from '../../../../misc/normalize-for-search';
 
 /*
 トレンドに載るためには「『直近a分間のユニーク投稿数が今からa分前～今からb分前の間のユニーク投稿数のn倍以上』のハッシュタグの上位5位以内に入る」ことが必要
@@ -54,7 +55,7 @@ export const meta = {
 
 export default define(meta, async () => {
 	const instance = await fetchMeta(true);
-	const hiddenTags = instance.hiddenTags.map(t => t.toLowerCase());
+	const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t));
 
 	const now = new Date(); // 5分単位で丸めた現在日時
 	now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0);

src/server/api/endpoints/hashtags/users.ts

@@ -1,6 +1,7 @@
 import $ from 'cafy';
 import define from '../../define';
 import { Users } from '../../../../models';
+import { normalizeForSearch } from '../../../../misc/normalize-for-search';
 
 export const meta = {
 	requireCredential: false as const,
@@ -59,7 +60,7 @@ export const meta = {
 
 export default define(meta, async (ps, me) => {
 	const query = Users.createQueryBuilder('user')
-		.where(':tag = ANY(user.tags)', { tag: ps.tag.toLowerCase() });
+		.where(':tag = ANY(user.tags)', { tag: normalizeForSearch(ps.tag) });
 
 	const recent = new Date(Date.now() - (1000 * 60 * 60 * 24 * 5));
 

src/server/api/endpoints/i/update.ts

@@ -15,6 +15,7 @@ import { User } from '../../../../models/entities/user';
 import { UserProfile } from '../../../../models/entities/user-profile';
 import { ensure } from '../../../../prelude/ensure';
 import { notificationTypes } from '../../../../types';
+import { normalizeForSearch } from '../../../../misc/normalize-for-search';
 
 export const meta = {
 	desc: {
@@ -286,7 +287,7 @@ export default define(meta, async (ps, user, token) => {
 	if (newDescription != null) {
 		const tokens = parse(newDescription);
 		emojis = emojis.concat(extractEmojis(tokens!));
-		tags = extractHashtags(tokens!).map(tag => tag.toLowerCase()).splice(0, 32);
+		tags = extractHashtags(tokens!).map(tag => normalizeForSearch(tag)).splice(0, 32);
 	}
 
 	updates.emojis = emojis;

src/server/api/endpoints/notes/search-by-tag.ts

@@ -7,6 +7,7 @@ import { generateMutedUserQuery } from '../../common/generate-muted-user-query';
 import { generateVisibilityQuery } from '../../common/generate-visibility-query';
 import { Brackets } from 'typeorm';
 import { safeForSql } from '../../../../misc/safe-for-sql';
+import { normalizeForSearch } from '../../../../misc/normalize-for-search';
 
 export const meta = {
 	desc: {
@@ -101,7 +102,7 @@ export default define(meta, async (ps, me) => {
 
 	if (ps.tag) {
 		if (!safeForSql(ps.tag)) return;
-		query.andWhere(`'{"${ps.tag.toLowerCase()}"}' <@ note.tags`);
+		query.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`);
 	} else {
 		let i = 0;
 		query.andWhere(new Brackets(qb => {
@@ -109,7 +110,7 @@ export default define(meta, async (ps, me) => {
 				qb.orWhere(new Brackets(qb => {
 					for (const tag of tags) {
 						if (!safeForSql(tag)) return;
-						qb.andWhere(`'{"${tag.toLowerCase()}"}' <@ note.tags`);
+						qb.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`);
 						i++;
 					}
 				}));

src/server/api/stream/channels/hashtag.ts

@@ -3,6 +3,7 @@ import { isMutedUserRelated } from '../../../../misc/is-muted-user-related';
 import Channel from '../channel';
 import { Notes } from '../../../../models';
 import { PackedNote } from '../../../../models/repositories/note';
+import { normalizeForSearch } from '../../../../misc/normalize-for-search';
 
 export default class extends Channel {
 	public readonly chName = 'hashtag';
@@ -23,7 +24,7 @@ export default class extends Channel {
 	@autobind
 	private async onNote(note: PackedNote) {
 		const noteTags = note.tags ? note.tags.map((t: string) => t.toLowerCase()) : [];
-		const matched = this.q.some(tags => tags.every(tag => noteTags.includes(tag.toLowerCase())));
+		const matched = this.q.some(tags => tags.every(tag => noteTags.includes(normalizeForSearch(tag))));
 		if (!matched) return;
 
 		// Renoteなら再pack

src/services/note/create.ts

@@ -33,6 +33,7 @@ import { addNoteToAntenna } from '../add-note-to-antenna';
 import { countSameRenotes } from '../../misc/count-same-renotes';
 import { deliverToRelays } from '../relay';
 import { Channel } from '../../models/entities/channel';
+import { normalizeForSearch } from '../../misc/normalize-for-search';
 
 type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
 
@@ -460,7 +461,7 @@ async function insertNote(user: User, data: Option, tags: string[], emojis: stri
 		text: data.text,
 		hasPoll: data.poll != null,
 		cw: data.cw == null ? null : data.cw,
-		tags: tags.map(tag => tag.toLowerCase()),
+		tags: tags.map(tag => normalizeForSearch(tag)),
 		emojis,
 		userId: user.id,
 		viaMobile: data.viaMobile!,
@@ -547,7 +548,7 @@ function index(note: Note) {
 		index: config.elasticsearch.index || 'misskey_note',
 		id: note.id.toString(),
 		body: {
-			text: note.text.toLowerCase(),
+			text: normalizeForSearch(note.text),
 			userId: note.userId,
 			userHost: note.userHost
 		}

src/services/update-hashtag.ts

@@ -3,6 +3,7 @@ import { Hashtags, Users } from '../models';
 import { hashtagChart } from './chart';
 import { genId } from '../misc/gen-id';
 import { Hashtag } from '../models/entities/hashtag';
+import { normalizeForSearch } from '../misc/normalize-for-search';
 
 export async function updateHashtags(user: User, tags: string[]) {
 	for (const tag of tags) {
@@ -21,7 +22,7 @@ export async function updateUsertags(user: User, tags: string[]) {
 }
 
 export async function updateHashtag(user: User, tag: string, isUserAttached = false, inc = true) {
-	tag = tag.toLowerCase();
+	tag = normalizeForSearch(tag);
 
 	const index = await Hashtags.findOne({ name: tag });