From 3f5b96bf629da5f736c09b10058802eed28cca18 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 16 May 2019 01:07:32 +0900 Subject: [PATCH] Resolve #4928 --- .circleci/misskey/default.yml | 2 - .circleci/misskey/test.yml | 2 - .config/example.yml | 55 ----- locales/ja-JP.yml | 13 + .../1557932705754-ObjectStorageSetting.ts | 31 +++ src/client/app/admin/views/instance.vue | 224 ++++++++++++------ src/config/types.ts | 7 - src/models/entities/meta.ts | 57 +++++ src/server/api/endpoints/admin/update-meta.ts | 82 ++++++- src/server/api/endpoints/meta.ts | 12 +- src/services/drive/add-file.ts | 27 ++- src/services/drive/delete-file.ts | 18 +- 12 files changed, 370 insertions(+), 160 deletions(-) create mode 100644 migration/1557932705754-ObjectStorageSetting.ts diff --git a/.circleci/misskey/default.yml b/.circleci/misskey/default.yml index c842431d24..5cdb7330c6 100644 --- a/.circleci/misskey/default.yml +++ b/.circleci/misskey/default.yml @@ -6,8 +6,6 @@ mongodb: db: misskey user: syuilo pass: '' -drive: - storage: 'db' redis: host: localhost port: 6379 diff --git a/.circleci/misskey/test.yml b/.circleci/misskey/test.yml index 450c5a79d8..99ad50876d 100644 --- a/.circleci/misskey/test.yml +++ b/.circleci/misskey/test.yml @@ -6,8 +6,6 @@ mongodb: db: test-misskey user: admin pass: '' -drive: - storage: 'db' # __REDIS__ redis: host: localhost diff --git a/.config/example.yml b/.config/example.yml index db278ecc27..0babd037c5 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -78,61 +78,6 @@ redis: # port: 9200 # pass: null -# ┌────────────────────────────────────┐ -#───┘ File storage (Drive) configuration └────────────────────── - -drive: - storage: 'fs' - -# OR - -#drive: -# storage: 'minio' -# bucket: -# prefix: -# config: -# endPoint: -# port: -# useSSL: -# accessKey: -# secretKey: - -# S3/GCS example -# -# * Replace to -# S3: see https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region -# GCS: use 'storage.googleapis.com' -# -# * Replace to -# S3: see https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region -# GCS: not needed (just delete the region line) -# -#drive: -# storage: 'minio' -# bucket: bucket-name -# prefix: files -# baseUrl: https://bucket-name. -# config: -# endPoint: -# region: -# useSSL: true -# accessKey: XXX -# secretKey: YYY - -# S3/GCS example (with CDN, custom domain) -# -#drive: -# storage: 'minio' -# bucket: drive.example.com -# prefix: files -# baseUrl: https://drive.example.com -# config: -# endPoint: -# region: -# useSSL: true -# accessKey: XXX -# secretKey: YYY - # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 76c1ab8269..bb991459ca 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1232,6 +1232,19 @@ admin/views/instance.vue: advanced-config: "その他の設定" note-and-tl: "投稿とタイムライン" drive-config: "ドライブの設定" + use-object-storage: "オブジェクトストレージを使用する" + object-storage-base-url: "URL" + object-storage-bucket: "バケット名" + object-storage-prefix: "プレフィックス" + object-storage-endpoint: "エンドポイント" + object-storage-region: "リージョン" + object-storage-port: "ポート" + object-storage-access-key: "アクセスキー" + object-storage-secret-key: "シークレットキー" + object-storage-use-ssl: "SSLを使用" + object-storage-s3-info: "Amazon S3をオブジェクトストレージとして使用する場合の「エンドポイント」と「リージョン」の設定については{0}をご確認ください。" + object-storage-s3-info-here: "こちら" + object-storage-gcs-info: "Google Cloud Storageをオブジェクトストレージとして使用する場合、「エンドポイント」は storage.googleapis.com に設定し、「リージョン」は空欄にします。" cache-remote-files: "リモートのファイルをキャッシュする" cache-remote-files-desc: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。そのためサーバーのストレージを節約できますが、プライバシー設定で直リンクを無効にしているユーザーにはファイルが見えなくなったり、サムネイルが生成されないので通信量が増加します。通常はこの設定をオンにしておくことをおすすめします。" local-drive-capacity-mb: "ローカルユーザーひとりあたりのドライブ容量" diff --git a/migration/1557932705754-ObjectStorageSetting.ts b/migration/1557932705754-ObjectStorageSetting.ts new file mode 100644 index 0000000000..dde6aa65f9 --- /dev/null +++ b/migration/1557932705754-ObjectStorageSetting.ts @@ -0,0 +1,31 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class ObjectStorageSetting1557932705754 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" ADD "useObjectStorage" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageBucket" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStoragePrefix" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageBaseUrl" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageEndpoint" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRegion" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageAccessKey" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageSecretKey" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStoragePort" integer`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageUseSSL" boolean NOT NULL DEFAULT true`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageUseSSL"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStoragePort"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageSecretKey"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageAccessKey"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRegion"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageEndpoint"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageBaseUrl"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStoragePrefix"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageBucket"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "useObjectStorage"`); + } + +} diff --git a/src/client/app/admin/views/instance.vue b/src/client/app/admin/views/instance.vue index be9e56131e..3ac4d6d721 100644 --- a/src/client/app/admin/views/instance.vue +++ b/src/client/app/admin/views/instance.vue @@ -53,6 +53,32 @@ +
+ {{ $t('use-object-storage') }} + +
{{ $t('cache-remote-files') }}
@@ -65,69 +91,6 @@ - - -
- {{ $t('enable-email') }} - {{ $t('email') }} - - {{ $t('smtp-host') }} - {{ $t('smtp-port') }} - - {{ $t('smtp-auth') }} - - {{ $t('smtp-user') }} - {{ $t('smtp-pass') }} - - {{ $t('smtp-secure') }} -
-
- {{ $t('save') }} -
- - - - -
- {{ $t('proxy-account-info') }} - {{ $t('proxy-account-username') }} - {{ $t('proxy-account-warn') }} -
-
- {{ $t('save') }} -
- - - - -
- {{ $t('enable-serviceworker') }} - {{ $t('vapid-info') }}
npm i web-push -g
web-push generate-vapid-keys - - {{ $t('vapid-publickey') }} - {{ $t('vapid-privatekey') }} - -
-
- {{ $t('save') }} -
- - - - -
- {{ $t('enable-recaptcha') }} - {{ $t('recaptcha-info') }} - - {{ $t('recaptcha-site-key') }} - {{ $t('recaptcha-secret-key') }} - -
-
- {{ $t('save') }} -
- -
@@ -138,34 +101,109 @@
+ + +
+ {{ $t('proxy-account-info') }} + {{ $t('proxy-account-username') }} + {{ $t('proxy-account-warn') }} +
+
+ {{ $t('save') }} +
+ + + + +
+ {{ $t('enable-email') }} + +
+
+ {{ $t('save') }} +
+ + + + +
+ {{ $t('enable-serviceworker') }} + +
+
+ {{ $t('save') }} +
+ + + + +
+ {{ $t('enable-recaptcha') }} + +
+
+ {{ $t('save') }} +
+ +
{{ $t('twitter-integration-config') }}
{{ $t('enable-twitter-integration') }} - - {{ $t('twitter-integration-consumer-key') }} - {{ $t('twitter-integration-consumer-secret') }} - - {{ $t('twitter-integration-info', { url: `${url}/api/tw/cb` }) }} +
{{ $t('github-integration-config') }}
{{ $t('enable-github-integration') }} - - {{ $t('github-integration-client-id') }} - {{ $t('github-integration-client-secret') }} - - {{ $t('github-integration-info', { url: `${url}/api/gh/cb` }) }} +
{{ $t('discord-integration-config') }}
{{ $t('enable-discord-integration') }} - - {{ $t('discord-integration-client-id') }} - {{ $t('discord-integration-client-secret') }} - - {{ $t('discord-integration-info', { url: `${url}/api/dc/cb` }) }} +
{{ $t('save') }} @@ -261,6 +299,16 @@ export default Vue.extend({ swPrivateKey: null, pinnedUsers: '', hiddenTags: '', + useObjectStorage: false, + objectStorageBaseUrl: null, + objectStorageBucket: null, + objectStoragePrefix: null, + objectStorageEndpoint: null, + objectStorageRegion: null, + objectStoragePort: null, + objectStorageAccessKey: null, + objectStorageSecretKey: null, + objectStorageUseSSL: false, faHeadset, faShieldAlt, faGhost, faUserPlus, farEnvelope, faBolt, faThumbtack, faPencilAlt, faSave, faHashtag }; }, @@ -315,6 +363,16 @@ export default Vue.extend({ this.swPrivateKey = meta.swPrivateKey; this.pinnedUsers = meta.pinnedUsers.join('\n'); this.hiddenTags = meta.hiddenTags.join('\n'); + this.useObjectStorage = meta.useObjectStorage; + this.objectStorageBaseUrl = meta.objectStorageBaseUrl; + this.objectStorageBucket = meta.objectStorageBucket; + this.objectStoragePrefix = meta.objectStoragePrefix; + this.objectStorageEndpoint = meta.objectStorageEndpoint; + this.objectStorageRegion = meta.objectStorageRegion; + this.objectStoragePort = meta.objectStoragePort; + this.objectStorageAccessKey = meta.objectStorageAccessKey; + this.objectStorageSecretKey = meta.objectStorageSecretKey; + this.objectStorageUseSSL = meta.objectStorageUseSSL; }); }, @@ -382,6 +440,16 @@ export default Vue.extend({ swPrivateKey: this.swPrivateKey, pinnedUsers: this.pinnedUsers.split('\n'), hiddenTags: this.hiddenTags.split('\n'), + useObjectStorage: this.useObjectStorage, + objectStorageBaseUrl: this.objectStorageBaseUrl ? this.objectStorageBaseUrl : null, + objectStorageBucket: this.objectStorageBucket ? this.objectStorageBucket : null, + objectStoragePrefix: this.objectStoragePrefix ? this.objectStoragePrefix : null, + objectStorageEndpoint: this.objectStorageEndpoint ? this.objectStorageEndpoint : null, + objectStorageRegion: this.objectStorageRegion ? this.objectStorageRegion : null, + objectStoragePort: this.objectStoragePort ? this.objectStoragePort : null, + objectStorageAccessKey: this.objectStorageAccessKey ? this.objectStorageAccessKey : null, + objectStorageSecretKey: this.objectStorageSecretKey ? this.objectStorageSecretKey : null, + objectStorageUseSSL: this.objectStorageUseSSL, }).then(() => { this.$root.dialog({ type: 'success', diff --git a/src/config/types.ts b/src/config/types.ts index d312a5a181..7da9820f22 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -27,13 +27,6 @@ export type Source = { port: number; pass: string; }; - drive?: { - storage: string; - bucket?: string; - prefix?: string; - baseUrl?: string; - config?: any; - }; autoAdmin?: boolean; diff --git a/src/models/entities/meta.ts b/src/models/entities/meta.ts index c3797a9ed6..fdd2818238 100644 --- a/src/models/entities/meta.ts +++ b/src/models/entities/meta.ts @@ -288,4 +288,61 @@ export class Meta { nullable: true }) public feedbackUrl: string | null; + + @Column('boolean', { + default: false, + }) + public useObjectStorage: boolean; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageBucket: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStoragePrefix: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageBaseUrl: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageEndpoint: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageRegion: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageAccessKey: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageSecretKey: string | null; + + @Column('integer', { + nullable: true + }) + public objectStoragePort: number | null; + + @Column('boolean', { + default: true, + }) + public objectStorageUseSSL: boolean; } diff --git a/src/server/api/endpoints/admin/update-meta.ts b/src/server/api/endpoints/admin/update-meta.ts index e4f2e86aaa..8e98d203ff 100644 --- a/src/server/api/endpoints/admin/update-meta.ts +++ b/src/server/api/endpoints/admin/update-meta.ts @@ -357,7 +357,47 @@ export const meta = { desc: { 'ja-JP': 'フィードバックのURL' } - } + }, + + useObjectStorage: { + validator: $.optional.bool + }, + + objectStorageBaseUrl: { + validator: $.optional.nullable.str + }, + + objectStorageBucket: { + validator: $.optional.nullable.str + }, + + objectStoragePrefix: { + validator: $.optional.nullable.str + }, + + objectStorageEndpoint: { + validator: $.optional.nullable.str + }, + + objectStorageRegion: { + validator: $.optional.nullable.str + }, + + objectStoragePort: { + validator: $.optional.nullable.num + }, + + objectStorageAccessKey: { + validator: $.optional.nullable.str + }, + + objectStorageSecretKey: { + validator: $.optional.nullable.str + }, + + objectStorageUseSSL: { + validator: $.optional.bool + }, } }; @@ -560,6 +600,46 @@ export default define(meta, async (ps) => { set.feedbackUrl = ps.feedbackUrl; } + if (ps.useObjectStorage !== undefined) { + set.useObjectStorage = ps.useObjectStorage; + } + + if (ps.objectStorageBaseUrl !== undefined) { + set.objectStorageBaseUrl = ps.objectStorageBaseUrl; + } + + if (ps.objectStorageBucket !== undefined) { + set.objectStorageBucket = ps.objectStorageBucket; + } + + if (ps.objectStoragePrefix !== undefined) { + set.objectStoragePrefix = ps.objectStoragePrefix; + } + + if (ps.objectStorageEndpoint !== undefined) { + set.objectStorageEndpoint = ps.objectStorageEndpoint; + } + + if (ps.objectStorageRegion !== undefined) { + set.objectStorageRegion = ps.objectStorageRegion; + } + + if (ps.objectStoragePort !== undefined) { + set.objectStoragePort = ps.objectStoragePort; + } + + if (ps.objectStorageAccessKey !== undefined) { + set.objectStorageAccessKey = ps.objectStorageAccessKey; + } + + if (ps.objectStorageSecretKey !== undefined) { + set.objectStorageSecretKey = ps.objectStorageSecretKey; + } + + if (ps.objectStorageUseSSL !== undefined) { + set.objectStorageUseSSL = ps.objectStorageUseSSL; + } + await getConnection().transaction(async transactionalEntityManager => { const meta = await transactionalEntityManager.findOne(Meta, { order: { diff --git a/src/server/api/endpoints/meta.ts b/src/server/api/endpoints/meta.ts index 1bd88a1e6d..4f418c63c1 100644 --- a/src/server/api/endpoints/meta.ts +++ b/src/server/api/endpoints/meta.ts @@ -153,7 +153,7 @@ export default define(meta, async (ps, me) => { globalTimeLine: !instance.disableGlobalTimeline, elasticsearch: config.elasticsearch ? true : false, recaptcha: instance.enableRecaptcha, - objectStorage: config.drive && config.drive.storage === 'minio', + objectStorage: instance.useObjectStorage, twitter: instance.enableTwitterIntegration, github: instance.enableGithubIntegration, discord: instance.enableDiscordIntegration, @@ -182,6 +182,16 @@ export default define(meta, async (ps, me) => { response.smtpUser = instance.smtpUser; response.smtpPass = instance.smtpPass; response.swPrivateKey = instance.swPrivateKey; + response.useObjectStorage = instance.useObjectStorage; + response.objectStorageBaseUrl = instance.objectStorageBaseUrl; + response.objectStorageBucket = instance.objectStorageBucket; + response.objectStoragePrefix = instance.objectStoragePrefix; + response.objectStorageEndpoint = instance.objectStorageEndpoint; + response.objectStorageRegion = instance.objectStorageRegion; + response.objectStoragePort = instance.objectStoragePort; + response.objectStorageAccessKey = instance.objectStorageAccessKey; + response.objectStorageSecretKey = instance.objectStorageSecretKey; + response.objectStorageUseSSL = instance.objectStorageUseSSL; } return response; diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index 949089eded..701878b282 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -8,7 +8,6 @@ import * as sharp from 'sharp'; import { publishMainStream, publishDriveStream } from '../stream'; import delFile from './delete-file'; -import config from '../../config'; import { fetchMeta } from '../../misc/fetch-meta'; import { GenerateVideoThumbnail } from './generate-video-thumbnail'; import { driveLogger } from './logger'; @@ -37,7 +36,9 @@ async function save(file: DriveFile, path: string, name: string, type: string, h // thunbnail, webpublic を必要なら生成 const alts = await generateAlts(path, type, !file.uri); - if (config.drive && config.drive.storage == 'minio') { + const meta = await fetchMeta(); + + if (meta.useObjectStorage) { //#region ObjectStorage params let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']); @@ -47,11 +48,11 @@ async function save(file: DriveFile, path: string, name: string, type: string, h if (type === 'image/webp') ext = '.webp'; } - const baseUrl = config.drive.baseUrl - || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; + const baseUrl = meta.objectStorageBaseUrl + || `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`; // for original - const key = `${config.drive.prefix}/${uuid.v4()}${ext}`; + const key = `${meta.objectStoragePrefix}/${uuid.v4()}${ext}`; const url = `${ baseUrl }/${ key }`; // for alts @@ -68,7 +69,7 @@ async function save(file: DriveFile, path: string, name: string, type: string, h ]; if (alts.webpublic) { - webpublicKey = `${config.drive.prefix}/${uuid.v4()}.${alts.webpublic.ext}`; + webpublicKey = `${meta.objectStoragePrefix}/${uuid.v4()}.${alts.webpublic.ext}`; webpublicUrl = `${ baseUrl }/${ webpublicKey }`; logger.info(`uploading webpublic: ${webpublicKey}`); @@ -76,7 +77,7 @@ async function save(file: DriveFile, path: string, name: string, type: string, h } if (alts.thumbnail) { - thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${alts.thumbnail.ext}`; + thumbnailKey = `${meta.objectStoragePrefix}/${uuid.v4()}.${alts.thumbnail.ext}`; thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`; logger.info(`uploading thumbnail: ${thumbnailKey}`); @@ -194,7 +195,15 @@ export async function generateAlts(path: string, type: string, generateWeb: bool * Upload to ObjectStorage */ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string, filename?: string) { - const minio = new Minio.Client(config.drive!.config); + const meta = await fetchMeta(); + + const minio = new Minio.Client({ + endPoint: meta.objectStorageEndpoint!, + port: meta.objectStoragePort ? meta.objectStoragePort : undefined, + useSSL: meta.objectStorageUseSSL, + accessKey: meta.objectStorageAccessKey!, + secretKey: meta.objectStorageSecretKey!, + }); const metadata = { 'Content-Type': type, @@ -203,7 +212,7 @@ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string, if (filename) metadata['Content-Disposition'] = contentDisposition('inline', filename); - await minio.putObject(config.drive!.bucket!, key, stream, undefined, metadata); + await minio.putObject(meta.objectStorageBucket!, key, stream, undefined, metadata); } async function deleteOldFile(user: IRemoteUser) { diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index f1280822a4..ba0482dbe2 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -1,9 +1,9 @@ import * as Minio from 'minio'; -import config from '../../config'; import { DriveFile } from '../../models/entities/drive-file'; import { InternalStorage } from './internal-storage'; import { DriveFiles, Instances, Notes } from '../../models'; import { driveChart, perUserDriveChart, instanceChart } from '../chart'; +import { fetchMeta } from '../../misc/fetch-meta'; export default async function(file: DriveFile, isExpired = false) { if (file.storedInternal) { @@ -17,16 +17,24 @@ export default async function(file: DriveFile, isExpired = false) { InternalStorage.del(file.webpublicAccessKey!); } } else if (!file.isLink) { - const minio = new Minio.Client(config.drive!.config); + const meta = await fetchMeta(); - await minio.removeObject(config.drive!.bucket!, file.accessKey!); + const minio = new Minio.Client({ + endPoint: meta.objectStorageEndpoint!, + port: meta.objectStoragePort ? meta.objectStoragePort : undefined, + useSSL: meta.objectStorageUseSSL, + accessKey: meta.objectStorageAccessKey!, + secretKey: meta.objectStorageSecretKey!, + }); + + await minio.removeObject(meta.objectStorageBucket!, file.accessKey!); if (file.thumbnailUrl) { - await minio.removeObject(config.drive!.bucket!, file.thumbnailAccessKey!); + await minio.removeObject(meta.objectStorageBucket!, file.thumbnailAccessKey!); } if (file.webpublicUrl) { - await minio.removeObject(config.drive!.bucket!, file.webpublicAccessKey!); + await minio.removeObject(meta.objectStorageBucket!, file.webpublicAccessKey!); } }