parent
050787238b
commit
658901a47f
|
@ -37,6 +37,9 @@
|
||||||
"@tensorflow/tfjs-node": "4.2.0"
|
"@tensorflow/tfjs-node": "4.2.0"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@aws-sdk/client-s3": "^3.294.0",
|
||||||
|
"@aws-sdk/lib-storage": "^3.294.0",
|
||||||
|
"@aws-sdk/node-http-handler": "^3.292.0",
|
||||||
"@bull-board/api": "5.0.0",
|
"@bull-board/api": "5.0.0",
|
||||||
"@bull-board/fastify": "5.0.0",
|
"@bull-board/fastify": "5.0.0",
|
||||||
"@bull-board/ui": "5.0.0",
|
"@bull-board/ui": "5.0.0",
|
||||||
|
@ -59,7 +62,6 @@
|
||||||
"ajv": "8.12.0",
|
"ajv": "8.12.0",
|
||||||
"archiver": "5.3.1",
|
"archiver": "5.3.1",
|
||||||
"autwh": "0.1.0",
|
"autwh": "0.1.0",
|
||||||
"aws-sdk": "2.1318.0",
|
|
||||||
"bcryptjs": "2.4.3",
|
"bcryptjs": "2.4.3",
|
||||||
"blurhash": "2.0.5",
|
"blurhash": "2.0.5",
|
||||||
"bull": "4.10.4",
|
"bull": "4.10.4",
|
||||||
|
@ -190,6 +192,7 @@
|
||||||
"@types/ws": "8.5.4",
|
"@types/ws": "8.5.4",
|
||||||
"@typescript-eslint/eslint-plugin": "5.54.1",
|
"@typescript-eslint/eslint-plugin": "5.54.1",
|
||||||
"@typescript-eslint/parser": "5.54.1",
|
"@typescript-eslint/parser": "5.54.1",
|
||||||
|
"aws-sdk-client-mock": "^2.1.1",
|
||||||
"cross-env": "7.0.3",
|
"cross-env": "7.0.3",
|
||||||
"eslint": "8.35.0",
|
"eslint": "8.35.0",
|
||||||
"eslint-plugin-import": "2.27.5",
|
"eslint-plugin-import": "2.27.5",
|
||||||
|
|
|
@ -4,6 +4,7 @@ import { v4 as uuid } from 'uuid';
|
||||||
import sharp from 'sharp';
|
import sharp from 'sharp';
|
||||||
import { sharpBmp } from 'sharp-read-bmp';
|
import { sharpBmp } from 'sharp-read-bmp';
|
||||||
import { IsNull } from 'typeorm';
|
import { IsNull } from 'typeorm';
|
||||||
|
import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3';
|
||||||
import { DI } from '@/di-symbols.js';
|
import { DI } from '@/di-symbols.js';
|
||||||
import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js';
|
import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js';
|
||||||
import type { Config } from '@/config.js';
|
import type { Config } from '@/config.js';
|
||||||
|
@ -36,7 +37,6 @@ import { bindThis } from '@/decorators.js';
|
||||||
import { RoleService } from '@/core/RoleService.js';
|
import { RoleService } from '@/core/RoleService.js';
|
||||||
import { correctFilename } from '@/misc/correct-filename.js';
|
import { correctFilename } from '@/misc/correct-filename.js';
|
||||||
import { isMimeImage } from '@/misc/is-mime-image.js';
|
import { isMimeImage } from '@/misc/is-mime-image.js';
|
||||||
import type S3 from 'aws-sdk/clients/s3.js';
|
|
||||||
|
|
||||||
type AddFileArgs = {
|
type AddFileArgs = {
|
||||||
/** User who wish to add file */
|
/** User who wish to add file */
|
||||||
|
@ -81,6 +81,7 @@ type UploadFromUrlArgs = {
|
||||||
export class DriveService {
|
export class DriveService {
|
||||||
private registerLogger: Logger;
|
private registerLogger: Logger;
|
||||||
private downloaderLogger: Logger;
|
private downloaderLogger: Logger;
|
||||||
|
private deleteLogger: Logger;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
@Inject(DI.config)
|
||||||
|
@ -118,6 +119,7 @@ export class DriveService {
|
||||||
const logger = new Logger('drive', 'blue');
|
const logger = new Logger('drive', 'blue');
|
||||||
this.registerLogger = logger.createSubLogger('register', 'yellow');
|
this.registerLogger = logger.createSubLogger('register', 'yellow');
|
||||||
this.downloaderLogger = logger.createSubLogger('downloader');
|
this.downloaderLogger = logger.createSubLogger('downloader');
|
||||||
|
this.deleteLogger = logger.createSubLogger('delete');
|
||||||
}
|
}
|
||||||
|
|
||||||
/***
|
/***
|
||||||
|
@ -368,7 +370,7 @@ export class DriveService {
|
||||||
Body: stream,
|
Body: stream,
|
||||||
ContentType: type,
|
ContentType: type,
|
||||||
CacheControl: 'max-age=31536000, immutable',
|
CacheControl: 'max-age=31536000, immutable',
|
||||||
} as S3.PutObjectRequest;
|
} as PutObjectCommandInput;
|
||||||
|
|
||||||
if (filename) params.ContentDisposition = contentDisposition(
|
if (filename) params.ContentDisposition = contentDisposition(
|
||||||
'inline',
|
'inline',
|
||||||
|
@ -378,21 +380,16 @@ export class DriveService {
|
||||||
);
|
);
|
||||||
if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
|
if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
|
||||||
|
|
||||||
const s3 = this.s3Service.getS3(meta);
|
await this.s3Service.upload(meta, params)
|
||||||
|
|
||||||
const upload = s3.upload(params, {
|
|
||||||
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
|
|
||||||
});
|
|
||||||
|
|
||||||
await upload.promise()
|
|
||||||
.then(
|
.then(
|
||||||
result => {
|
result => {
|
||||||
if (result) {
|
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
|
||||||
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
|
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
|
||||||
} else {
|
} else { // AbortMultipartUploadCommandOutput
|
||||||
this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`);
|
this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
|
||||||
}
|
}
|
||||||
},
|
})
|
||||||
|
.catch(
|
||||||
err => {
|
err => {
|
||||||
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
|
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
|
||||||
},
|
},
|
||||||
|
@ -528,10 +525,10 @@ export class DriveService {
|
||||||
};
|
};
|
||||||
|
|
||||||
const properties: {
|
const properties: {
|
||||||
width?: number;
|
width?: number;
|
||||||
height?: number;
|
height?: number;
|
||||||
orientation?: number;
|
orientation?: number;
|
||||||
} = {};
|
} = {};
|
||||||
|
|
||||||
if (info.width) {
|
if (info.width) {
|
||||||
properties['width'] = info.width;
|
properties['width'] = info.width;
|
||||||
|
@ -720,22 +717,22 @@ export class DriveService {
|
||||||
@bindThis
|
@bindThis
|
||||||
public async deleteObjectStorageFile(key: string) {
|
public async deleteObjectStorageFile(key: string) {
|
||||||
const meta = await this.metaService.fetch();
|
const meta = await this.metaService.fetch();
|
||||||
|
|
||||||
const s3 = this.s3Service.getS3(meta);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await s3.deleteObject({
|
const param = {
|
||||||
Bucket: meta.objectStorageBucket!,
|
Bucket: meta.objectStorageBucket,
|
||||||
Key: key,
|
Key: key,
|
||||||
}).promise();
|
} as DeleteObjectCommandInput;
|
||||||
|
|
||||||
|
await this.s3Service.delete(meta, param);
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
if (err.code === 'NoSuchKey') {
|
if (err.name === 'NoSuchKey') {
|
||||||
console.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err);
|
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
|
||||||
return;
|
return;
|
||||||
|
} else {
|
||||||
|
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
|
||||||
|
cause: err,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
|
|
||||||
cause: err,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,11 +1,16 @@
|
||||||
import { URL } from 'node:url';
|
import { URL } from 'node:url';
|
||||||
|
import * as http from 'node:http';
|
||||||
|
import * as https from 'node:https';
|
||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
import S3 from 'aws-sdk/clients/s3.js';
|
import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||||
|
import { Upload } from '@aws-sdk/lib-storage';
|
||||||
|
import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler';
|
||||||
import { DI } from '@/di-symbols.js';
|
import { DI } from '@/di-symbols.js';
|
||||||
import type { Config } from '@/config.js';
|
import type { Config } from '@/config.js';
|
||||||
import type { Meta } from '@/models/entities/Meta.js';
|
import type { Meta } from '@/models/entities/Meta.js';
|
||||||
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
|
import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class S3Service {
|
export class S3Service {
|
||||||
|
@ -18,25 +23,47 @@ export class S3Service {
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public getS3(meta: Meta) {
|
public getS3Client(meta: Meta): S3Client {
|
||||||
const u = meta.objectStorageEndpoint
|
const u = meta.objectStorageEndpoint
|
||||||
? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}`
|
? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
|
||||||
: `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`;
|
: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
|
||||||
|
|
||||||
return new S3({
|
const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
|
||||||
endpoint: meta.objectStorageEndpoint && meta.objectStorageEndpoint.length > 0
|
const handlerOption: NodeHttpHandlerOptions = {};
|
||||||
? meta.objectStorageEndpoint
|
if (meta.objectStorageUseSSL) {
|
||||||
: undefined,
|
handlerOption.httpsAgent = agent as https.Agent;
|
||||||
accessKeyId: meta.objectStorageAccessKey!,
|
} else {
|
||||||
secretAccessKey: meta.objectStorageSecretKey!,
|
handlerOption.httpAgent = agent as http.Agent;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new S3Client({
|
||||||
|
endpoint: meta.objectStorageEndpoint ? u : undefined,
|
||||||
|
credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
|
||||||
|
accessKeyId: meta.objectStorageAccessKey,
|
||||||
|
secretAccessKey: meta.objectStorageSecretKey,
|
||||||
|
} : undefined,
|
||||||
region: meta.objectStorageRegion ?? undefined,
|
region: meta.objectStorageRegion ?? undefined,
|
||||||
sslEnabled: meta.objectStorageUseSSL,
|
tls: meta.objectStorageUseSSL,
|
||||||
s3ForcePathStyle: !meta.objectStorageEndpoint // AWS with endPoint omitted
|
forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
|
||||||
? false
|
requestHandler: new NodeHttpHandler(handlerOption),
|
||||||
: meta.objectStorageS3ForcePathStyle,
|
|
||||||
httpOptions: {
|
|
||||||
agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public async upload(meta: Meta, input: PutObjectCommandInput) {
|
||||||
|
const client = this.getS3Client(meta);
|
||||||
|
return new Upload({
|
||||||
|
client,
|
||||||
|
params: input,
|
||||||
|
partSize: (client.config.endpoint && (await client.config.endpoint()).hostname === 'storage.googleapis.com')
|
||||||
|
? 500 * 1024 * 1024
|
||||||
|
: 8 * 1024 * 1024,
|
||||||
|
}).done();
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public delete(meta: Meta, input: DeleteObjectCommandInput) {
|
||||||
|
const client = this.getS3Client(meta);
|
||||||
|
return client.send(new DeleteObjectCommand(input));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,55 +1,56 @@
|
||||||
process.env.NODE_ENV = 'test';
|
process.env.NODE_ENV = 'test';
|
||||||
|
|
||||||
import { jest } from '@jest/globals';
|
|
||||||
import { Test } from '@nestjs/testing';
|
import { Test } from '@nestjs/testing';
|
||||||
|
import { DeleteObjectCommandOutput, DeleteObjectCommand, NoSuchKey, InvalidObjectState, S3Client } from '@aws-sdk/client-s3';
|
||||||
|
import { mockClient } from 'aws-sdk-client-mock';
|
||||||
import { GlobalModule } from '@/GlobalModule.js';
|
import { GlobalModule } from '@/GlobalModule.js';
|
||||||
import { DriveService } from '@/core/DriveService.js';
|
import { DriveService } from '@/core/DriveService.js';
|
||||||
import { CoreModule } from '@/core/CoreModule.js';
|
import { CoreModule } from '@/core/CoreModule.js';
|
||||||
import { S3Service } from '@/core/S3Service';
|
|
||||||
import type { Meta } from '@/models';
|
|
||||||
import type { DeleteObjectOutput } from 'aws-sdk/clients/s3';
|
|
||||||
import type { AWSError } from 'aws-sdk/lib/error';
|
|
||||||
import type { PromiseResult, Request } from 'aws-sdk/lib/request';
|
|
||||||
import type { TestingModule } from '@nestjs/testing';
|
import type { TestingModule } from '@nestjs/testing';
|
||||||
|
|
||||||
describe('DriveService', () => {
|
describe('DriveService', () => {
|
||||||
let app: TestingModule;
|
let app: TestingModule;
|
||||||
let driveService: DriveService;
|
let driveService: DriveService;
|
||||||
|
const s3Mock = mockClient(S3Client);
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeAll(async () => {
|
||||||
app = await Test.createTestingModule({
|
app = await Test.createTestingModule({
|
||||||
imports: [GlobalModule, CoreModule],
|
imports: [GlobalModule, CoreModule],
|
||||||
providers: [DriveService, S3Service],
|
providers: [DriveService],
|
||||||
}).compile();
|
}).compile();
|
||||||
app.enableShutdownHooks();
|
app.enableShutdownHooks();
|
||||||
driveService = app.get<DriveService>(DriveService);
|
driveService = app.get<DriveService>(DriveService);
|
||||||
|
});
|
||||||
|
|
||||||
const s3Service = app.get<S3Service>(S3Service);
|
beforeEach(async () => {
|
||||||
const s3 = s3Service.getS3({} as Meta);
|
s3Mock.reset();
|
||||||
|
});
|
||||||
|
|
||||||
// new S3() surprisingly does not return an instance of class S3.
|
afterAll(async () => {
|
||||||
// Let's use getPrototypeOf here to get a real prototype, since spying on S3.prototype doesn't work.
|
await app.close();
|
||||||
// TODO: Use `aws-sdk-client-mock` package when upgrading to AWS SDK v3.
|
|
||||||
jest.spyOn(Object.getPrototypeOf(s3), 'deleteObject').mockImplementation(() => {
|
|
||||||
// Roughly mock AWS request object
|
|
||||||
return {
|
|
||||||
async promise(): Promise<PromiseResult<DeleteObjectOutput, AWSError>> {
|
|
||||||
const err = new Error('mock') as AWSError;
|
|
||||||
err.code = 'NoSuchKey';
|
|
||||||
throw err;
|
|
||||||
},
|
|
||||||
} as Request<DeleteObjectOutput, AWSError>;
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('Object storage', () => {
|
describe('Object storage', () => {
|
||||||
|
test('delete a file', async () => {
|
||||||
|
s3Mock.on(DeleteObjectCommand)
|
||||||
|
.resolves({} as DeleteObjectCommandOutput);
|
||||||
|
|
||||||
|
await driveService.deleteObjectStorageFile('peace of the world');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('delete a file then unexpected error', async () => {
|
||||||
|
s3Mock.on(DeleteObjectCommand)
|
||||||
|
.rejects(new InvalidObjectState({ $metadata: {}, message: '' }));
|
||||||
|
|
||||||
|
await expect(driveService.deleteObjectStorageFile('unexpected')).rejects.toThrowError(Error);
|
||||||
|
});
|
||||||
|
|
||||||
test('delete a file with no valid key', async () => {
|
test('delete a file with no valid key', async () => {
|
||||||
try {
|
// Some S3 implementations returns 404 Not Found on deleting with a non-existent key
|
||||||
await driveService.deleteObjectStorageFile('lol no way');
|
s3Mock.on(DeleteObjectCommand)
|
||||||
} catch (err: any) {
|
.rejects(new NoSuchKey({ $metadata: {}, message: 'allowed error.' }));
|
||||||
console.log(err.cause);
|
|
||||||
throw err;
|
await driveService.deleteObjectStorageFile('lol no way');
|
||||||
}
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -0,0 +1,77 @@
|
||||||
|
process.env.NODE_ENV = 'test';
|
||||||
|
|
||||||
|
import { Test } from '@nestjs/testing';
|
||||||
|
import { UploadPartCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
|
||||||
|
import { mockClient } from 'aws-sdk-client-mock';
|
||||||
|
import { GlobalModule } from '@/GlobalModule.js';
|
||||||
|
import { CoreModule } from '@/core/CoreModule.js';
|
||||||
|
import { S3Service } from '@/core/S3Service';
|
||||||
|
import { Meta } from '@/models';
|
||||||
|
import type { TestingModule } from '@nestjs/testing';
|
||||||
|
|
||||||
|
describe('S3Service', () => {
|
||||||
|
let app: TestingModule;
|
||||||
|
let s3Service: S3Service;
|
||||||
|
const s3Mock = mockClient(S3Client);
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
app = await Test.createTestingModule({
|
||||||
|
imports: [GlobalModule, CoreModule],
|
||||||
|
providers: [S3Service],
|
||||||
|
}).compile();
|
||||||
|
app.enableShutdownHooks();
|
||||||
|
s3Service = app.get<S3Service>(S3Service);
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
s3Mock.reset();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('upload', () => {
|
||||||
|
test('upload a file', async () => {
|
||||||
|
s3Mock.on(PutObjectCommand).resolves({});
|
||||||
|
|
||||||
|
await s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, {
|
||||||
|
Bucket: 'fake',
|
||||||
|
Key: 'fake',
|
||||||
|
Body: 'x',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('upload a large file', async () => {
|
||||||
|
s3Mock.on(CreateMultipartUploadCommand).resolves({ UploadId: '1' });
|
||||||
|
s3Mock.on(UploadPartCommand).resolves({ ETag: '1' });
|
||||||
|
s3Mock.on(CompleteMultipartUploadCommand).resolves({ Bucket: 'fake', Key: 'fake' });
|
||||||
|
|
||||||
|
await s3Service.upload({} as Meta, {
|
||||||
|
Bucket: 'fake',
|
||||||
|
Key: 'fake',
|
||||||
|
Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('upload a file error', async () => {
|
||||||
|
s3Mock.on(PutObjectCommand).rejects({ name: 'Fake Error' });
|
||||||
|
|
||||||
|
await expect(s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, {
|
||||||
|
Bucket: 'fake',
|
||||||
|
Key: 'fake',
|
||||||
|
Body: 'x',
|
||||||
|
})).rejects.toThrowError(Error);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('upload a large file error', async () => {
|
||||||
|
s3Mock.on(UploadPartCommand).rejects();
|
||||||
|
|
||||||
|
await expect(s3Service.upload({} as Meta, {
|
||||||
|
Bucket: 'fake',
|
||||||
|
Key: 'fake',
|
||||||
|
Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ
|
||||||
|
})).rejects.toThrowError(Error);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
1176
pnpm-lock.yaml
1176
pnpm-lock.yaml
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue