bump aws-sdk to v3 for s3 (#10363)

* indent

* aws-sdk v3移行
This commit is contained in:
YS 2023-03-23 13:48:14 +09:00 committed by GitHub
parent 050787238b
commit 658901a47f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1267 additions and 164 deletions

View File

@ -37,6 +37,9 @@
"@tensorflow/tfjs-node": "4.2.0" "@tensorflow/tfjs-node": "4.2.0"
}, },
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.294.0",
"@aws-sdk/lib-storage": "^3.294.0",
"@aws-sdk/node-http-handler": "^3.292.0",
"@bull-board/api": "5.0.0", "@bull-board/api": "5.0.0",
"@bull-board/fastify": "5.0.0", "@bull-board/fastify": "5.0.0",
"@bull-board/ui": "5.0.0", "@bull-board/ui": "5.0.0",
@ -59,7 +62,6 @@
"ajv": "8.12.0", "ajv": "8.12.0",
"archiver": "5.3.1", "archiver": "5.3.1",
"autwh": "0.1.0", "autwh": "0.1.0",
"aws-sdk": "2.1318.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"blurhash": "2.0.5", "blurhash": "2.0.5",
"bull": "4.10.4", "bull": "4.10.4",
@ -190,6 +192,7 @@
"@types/ws": "8.5.4", "@types/ws": "8.5.4",
"@typescript-eslint/eslint-plugin": "5.54.1", "@typescript-eslint/eslint-plugin": "5.54.1",
"@typescript-eslint/parser": "5.54.1", "@typescript-eslint/parser": "5.54.1",
"aws-sdk-client-mock": "^2.1.1",
"cross-env": "7.0.3", "cross-env": "7.0.3",
"eslint": "8.35.0", "eslint": "8.35.0",
"eslint-plugin-import": "2.27.5", "eslint-plugin-import": "2.27.5",

View File

@ -4,6 +4,7 @@ import { v4 as uuid } from 'uuid';
import sharp from 'sharp'; import sharp from 'sharp';
import { sharpBmp } from 'sharp-read-bmp'; import { sharpBmp } from 'sharp-read-bmp';
import { IsNull } from 'typeorm'; import { IsNull } from 'typeorm';
import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js'; import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
@ -36,7 +37,6 @@ import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js'; import { RoleService } from '@/core/RoleService.js';
import { correctFilename } from '@/misc/correct-filename.js'; import { correctFilename } from '@/misc/correct-filename.js';
import { isMimeImage } from '@/misc/is-mime-image.js'; import { isMimeImage } from '@/misc/is-mime-image.js';
import type S3 from 'aws-sdk/clients/s3.js';
type AddFileArgs = { type AddFileArgs = {
/** User who wish to add file */ /** User who wish to add file */
@ -81,6 +81,7 @@ type UploadFromUrlArgs = {
export class DriveService { export class DriveService {
private registerLogger: Logger; private registerLogger: Logger;
private downloaderLogger: Logger; private downloaderLogger: Logger;
private deleteLogger: Logger;
constructor( constructor(
@Inject(DI.config) @Inject(DI.config)
@ -118,6 +119,7 @@ export class DriveService {
const logger = new Logger('drive', 'blue'); const logger = new Logger('drive', 'blue');
this.registerLogger = logger.createSubLogger('register', 'yellow'); this.registerLogger = logger.createSubLogger('register', 'yellow');
this.downloaderLogger = logger.createSubLogger('downloader'); this.downloaderLogger = logger.createSubLogger('downloader');
this.deleteLogger = logger.createSubLogger('delete');
} }
/*** /***
@ -368,7 +370,7 @@ export class DriveService {
Body: stream, Body: stream,
ContentType: type, ContentType: type,
CacheControl: 'max-age=31536000, immutable', CacheControl: 'max-age=31536000, immutable',
} as S3.PutObjectRequest; } as PutObjectCommandInput;
if (filename) params.ContentDisposition = contentDisposition( if (filename) params.ContentDisposition = contentDisposition(
'inline', 'inline',
@ -378,21 +380,16 @@ export class DriveService {
); );
if (meta.objectStorageSetPublicRead) params.ACL = 'public-read'; if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
const s3 = this.s3Service.getS3(meta); await this.s3Service.upload(meta, params)
const upload = s3.upload(params, {
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
});
await upload.promise()
.then( .then(
result => { result => {
if (result) { if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
} else { } else { // AbortMultipartUploadCommandOutput
this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`); this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
} }
}, })
.catch(
err => { err => {
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
}, },
@ -528,10 +525,10 @@ export class DriveService {
}; };
const properties: { const properties: {
width?: number; width?: number;
height?: number; height?: number;
orientation?: number; orientation?: number;
} = {}; } = {};
if (info.width) { if (info.width) {
properties['width'] = info.width; properties['width'] = info.width;
@ -720,22 +717,22 @@ export class DriveService {
@bindThis @bindThis
public async deleteObjectStorageFile(key: string) { public async deleteObjectStorageFile(key: string) {
const meta = await this.metaService.fetch(); const meta = await this.metaService.fetch();
const s3 = this.s3Service.getS3(meta);
try { try {
await s3.deleteObject({ const param = {
Bucket: meta.objectStorageBucket!, Bucket: meta.objectStorageBucket,
Key: key, Key: key,
}).promise(); } as DeleteObjectCommandInput;
await this.s3Service.delete(meta, param);
} catch (err: any) { } catch (err: any) {
if (err.code === 'NoSuchKey') { if (err.name === 'NoSuchKey') {
console.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err); this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
return; return;
} else {
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
cause: err,
});
} }
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
cause: err,
});
} }
} }

View File

@ -1,11 +1,16 @@
import { URL } from 'node:url'; import { URL } from 'node:url';
import * as http from 'node:http';
import * as https from 'node:https';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import S3 from 'aws-sdk/clients/s3.js'; import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { Upload } from '@aws-sdk/lib-storage';
import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import type { Meta } from '@/models/entities/Meta.js'; import type { Meta } from '@/models/entities/Meta.js';
import { HttpRequestService } from '@/core/HttpRequestService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3';
@Injectable() @Injectable()
export class S3Service { export class S3Service {
@ -18,25 +23,47 @@ export class S3Service {
} }
@bindThis @bindThis
public getS3(meta: Meta) { public getS3Client(meta: Meta): S3Client {
const u = meta.objectStorageEndpoint const u = meta.objectStorageEndpoint
? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}` ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
: `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`; : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
return new S3({ const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
endpoint: meta.objectStorageEndpoint && meta.objectStorageEndpoint.length > 0 const handlerOption: NodeHttpHandlerOptions = {};
? meta.objectStorageEndpoint if (meta.objectStorageUseSSL) {
: undefined, handlerOption.httpsAgent = agent as https.Agent;
accessKeyId: meta.objectStorageAccessKey!, } else {
secretAccessKey: meta.objectStorageSecretKey!, handlerOption.httpAgent = agent as http.Agent;
}
return new S3Client({
endpoint: meta.objectStorageEndpoint ? u : undefined,
credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
accessKeyId: meta.objectStorageAccessKey,
secretAccessKey: meta.objectStorageSecretKey,
} : undefined,
region: meta.objectStorageRegion ?? undefined, region: meta.objectStorageRegion ?? undefined,
sslEnabled: meta.objectStorageUseSSL, tls: meta.objectStorageUseSSL,
s3ForcePathStyle: !meta.objectStorageEndpoint // AWS with endPoint omitted forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
? false requestHandler: new NodeHttpHandler(handlerOption),
: meta.objectStorageS3ForcePathStyle,
httpOptions: {
agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
},
}); });
} }
@bindThis
public async upload(meta: Meta, input: PutObjectCommandInput) {
const client = this.getS3Client(meta);
return new Upload({
client,
params: input,
partSize: (client.config.endpoint && (await client.config.endpoint()).hostname === 'storage.googleapis.com')
? 500 * 1024 * 1024
: 8 * 1024 * 1024,
}).done();
}
@bindThis
public delete(meta: Meta, input: DeleteObjectCommandInput) {
const client = this.getS3Client(meta);
return client.send(new DeleteObjectCommand(input));
}
} }

View File

@ -1,55 +1,56 @@
process.env.NODE_ENV = 'test'; process.env.NODE_ENV = 'test';
import { jest } from '@jest/globals';
import { Test } from '@nestjs/testing'; import { Test } from '@nestjs/testing';
import { DeleteObjectCommandOutput, DeleteObjectCommand, NoSuchKey, InvalidObjectState, S3Client } from '@aws-sdk/client-s3';
import { mockClient } from 'aws-sdk-client-mock';
import { GlobalModule } from '@/GlobalModule.js'; import { GlobalModule } from '@/GlobalModule.js';
import { DriveService } from '@/core/DriveService.js'; import { DriveService } from '@/core/DriveService.js';
import { CoreModule } from '@/core/CoreModule.js'; import { CoreModule } from '@/core/CoreModule.js';
import { S3Service } from '@/core/S3Service';
import type { Meta } from '@/models';
import type { DeleteObjectOutput } from 'aws-sdk/clients/s3';
import type { AWSError } from 'aws-sdk/lib/error';
import type { PromiseResult, Request } from 'aws-sdk/lib/request';
import type { TestingModule } from '@nestjs/testing'; import type { TestingModule } from '@nestjs/testing';
describe('DriveService', () => { describe('DriveService', () => {
let app: TestingModule; let app: TestingModule;
let driveService: DriveService; let driveService: DriveService;
const s3Mock = mockClient(S3Client);
beforeEach(async () => { beforeAll(async () => {
app = await Test.createTestingModule({ app = await Test.createTestingModule({
imports: [GlobalModule, CoreModule], imports: [GlobalModule, CoreModule],
providers: [DriveService, S3Service], providers: [DriveService],
}).compile(); }).compile();
app.enableShutdownHooks(); app.enableShutdownHooks();
driveService = app.get<DriveService>(DriveService); driveService = app.get<DriveService>(DriveService);
});
const s3Service = app.get<S3Service>(S3Service); beforeEach(async () => {
const s3 = s3Service.getS3({} as Meta); s3Mock.reset();
});
// new S3() surprisingly does not return an instance of class S3. afterAll(async () => {
// Let's use getPrototypeOf here to get a real prototype, since spying on S3.prototype doesn't work. await app.close();
// TODO: Use `aws-sdk-client-mock` package when upgrading to AWS SDK v3.
jest.spyOn(Object.getPrototypeOf(s3), 'deleteObject').mockImplementation(() => {
// Roughly mock AWS request object
return {
async promise(): Promise<PromiseResult<DeleteObjectOutput, AWSError>> {
const err = new Error('mock') as AWSError;
err.code = 'NoSuchKey';
throw err;
},
} as Request<DeleteObjectOutput, AWSError>;
});
}); });
describe('Object storage', () => { describe('Object storage', () => {
test('delete a file', async () => {
s3Mock.on(DeleteObjectCommand)
.resolves({} as DeleteObjectCommandOutput);
await driveService.deleteObjectStorageFile('peace of the world');
});
test('delete a file then unexpected error', async () => {
s3Mock.on(DeleteObjectCommand)
.rejects(new InvalidObjectState({ $metadata: {}, message: '' }));
await expect(driveService.deleteObjectStorageFile('unexpected')).rejects.toThrowError(Error);
});
test('delete a file with no valid key', async () => { test('delete a file with no valid key', async () => {
try { // Some S3 implementations returns 404 Not Found on deleting with a non-existent key
await driveService.deleteObjectStorageFile('lol no way'); s3Mock.on(DeleteObjectCommand)
} catch (err: any) { .rejects(new NoSuchKey({ $metadata: {}, message: 'allowed error.' }));
console.log(err.cause);
throw err; await driveService.deleteObjectStorageFile('lol no way');
}
}); });
}); });
}); });

View File

@ -0,0 +1,77 @@
process.env.NODE_ENV = 'test';
import { Test } from '@nestjs/testing';
import { UploadPartCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
import { mockClient } from 'aws-sdk-client-mock';
import { GlobalModule } from '@/GlobalModule.js';
import { CoreModule } from '@/core/CoreModule.js';
import { S3Service } from '@/core/S3Service';
import { Meta } from '@/models';
import type { TestingModule } from '@nestjs/testing';
describe('S3Service', () => {
let app: TestingModule;
let s3Service: S3Service;
const s3Mock = mockClient(S3Client);
beforeAll(async () => {
app = await Test.createTestingModule({
imports: [GlobalModule, CoreModule],
providers: [S3Service],
}).compile();
app.enableShutdownHooks();
s3Service = app.get<S3Service>(S3Service);
});
beforeEach(async () => {
s3Mock.reset();
});
afterAll(async () => {
await app.close();
});
describe('upload', () => {
test('upload a file', async () => {
s3Mock.on(PutObjectCommand).resolves({});
await s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, {
Bucket: 'fake',
Key: 'fake',
Body: 'x',
});
});
test('upload a large file', async () => {
s3Mock.on(CreateMultipartUploadCommand).resolves({ UploadId: '1' });
s3Mock.on(UploadPartCommand).resolves({ ETag: '1' });
s3Mock.on(CompleteMultipartUploadCommand).resolves({ Bucket: 'fake', Key: 'fake' });
await s3Service.upload({} as Meta, {
Bucket: 'fake',
Key: 'fake',
Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ
});
});
test('upload a file error', async () => {
s3Mock.on(PutObjectCommand).rejects({ name: 'Fake Error' });
await expect(s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, {
Bucket: 'fake',
Key: 'fake',
Body: 'x',
})).rejects.toThrowError(Error);
});
test('upload a large file error', async () => {
s3Mock.on(UploadPartCommand).rejects();
await expect(s3Service.upload({} as Meta, {
Bucket: 'fake',
Key: 'fake',
Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ
})).rejects.toThrowError(Error);
});
});
});

File diff suppressed because it is too large Load Diff