OCR image captioning work

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "calckey",
-	"version": "12.119.0-calc.5.1",
+	"version": "12.119.0-calc.6",
 	"codename": "aqua",
 	"repository": {
 		"type": "git",

packages/backend/package.json

@@ -110,6 +110,7 @@
 		"summaly": "2.7.0",
 		"syslog-pro": "1.0.0",
 		"systeminformation": "5.12.11",
+		"tesseract.js": "^3.0.3",
 		"tinycolor2": "1.4.2",
 		"tmp": "0.2.1",
 		"ts-loader": "9.4.1",

packages/backend/src/server/api/endpoints.ts

@@ -113,6 +113,7 @@ import * as ep___drive from './endpoints/drive.js';
 import * as ep___drive_files from './endpoints/drive/files.js';
 import * as ep___drive_files_attachedNotes from './endpoints/drive/files/attached-notes.js';
 import * as ep___drive_files_checkExistence from './endpoints/drive/files/check-existence.js';
+import * as ep___drive_files_captionImage from './endpoints/drive/files/caption-image.js';
 import * as ep___drive_files_create from './endpoints/drive/files/create.js';
 import * as ep___drive_files_delete from './endpoints/drive/files/delete.js';
 import * as ep___drive_files_findByHash from './endpoints/drive/files/find-by-hash.js';
@@ -434,6 +435,7 @@ const eps = [
 	['drive', ep___drive],
 	['drive/files', ep___drive_files],
 	['drive/files/attached-notes', ep___drive_files_attachedNotes],
+	['drive/files/caption-image', ep___drive_files_captionImage],
 	['drive/files/check-existence', ep___drive_files_checkExistence],
 	['drive/files/create', ep___drive_files_create],
 	['drive/files/delete', ep___drive_files_delete],

packages/backend/src/server/api/endpoints/drive/files/caption-image.ts

@@ -0,0 +1,41 @@
+import define from '../../../define.js';
+import { createWorker } from 'tesseract.js';
+
+export const meta = {
+	tags: ['drive'],
+
+	requireCredential: true,
+
+	kind: 'read:drive',
+
+	description: 'Return caption of image',
+
+	res: {
+		type: 'string',
+		optional: false, nullable: false,
+	},
+} as const;
+
+export const paramDef = {
+	type: 'object',
+	properties: {
+		url: { type: 'string' },
+	},
+	required: ['url'],
+} as const;
+
+// eslint-disable-next-line import/no-default-export
+export default define(meta, paramDef, async (ps) => {
+
+	const worker = createWorker({
+		logger: m => console.log(m)
+	});
+
+	await worker.load();
+	await worker.loadLanguage('eng');
+	await worker.initialize('eng');
+	const { data: { text } } = await worker.recognize(ps.url);
+	await worker.terminate();
+
+	return text;
+});

packages/client/package.json

@@ -48,7 +48,6 @@
 		"stringz": "2.1.0",
 		"swiper": "^8.4.4",
 		"syuilo-password-strength": "0.0.1",
-		"tesseract.js": "^3.0.3",
 		"textarea-caret": "3.1.0",
 		"three": "0.144.0",
 		"throttle-debounce": "5.0.0",

packages/client/src/components/MkMediaCaption.vue

@@ -11,9 +11,9 @@
 				</header>
 				<textarea id="captioninput" v-model="inputValue" autofocus :placeholder="input.placeholder" @keydown="onInputKeydown"></textarea>
 				<div v-if="(showOkButton || showCaptionButton || showCancelButton)" class="buttons">
-					<MkButton inline primary :disabled="remainingLength < 0" @click="ok">{{ $ts.ok }}</MkButton>
-					<!-- <MkButton inline @click="caption" >{{ $ts.caption }}</MkButton> -->
-					<MkButton inline @click="cancel" >{{ $ts.cancel }}</MkButton>
+					<MkButton inline primary :disabled="remainingLength < 0" @click="ok">{{ i18n.ts.ok }}</MkButton>
+					<MkButton inline @click="caption">{{ i18n.ts.caption }}</MkButton>
+					<MkButton inline @click="cancel">{{ i18n.ts.cancel }}</MkButton>
 				</div>
 			</div>
 		</div>
@@ -33,7 +33,8 @@
 <script lang="ts">
 import { defineComponent } from 'vue';
 import { length } from 'stringz';
-import Tesseract from 'tesseract.js';
+import { i18n } from '@/i18n';
+import * as os from '@/os';
 import MkModal from '@/components/MkModal.vue';
 import MkButton from '@/components/MkButton.vue';
 import bytes from '@/filters/bytes';
@@ -141,37 +142,14 @@ export default defineComponent({
 		},
 
 		caption() {
-			const getBase64FromUrl = async (url) => {
-				const data = await fetch(url);
-				const blob = await data.blob();
-				return new Promise((resolve) => {
-					const reader = new FileReader();
-					reader.readAsDataURL(blob);
-					reader.onloadend = () => {
-						const base64data = reader.result;
-						resolve(base64data);
-					};
-				});
-			};
 			const img = document.getElementById('imgtocaption') as HTMLImageElement;
-			const b64 = getBase64FromUrl(img.src);
-			console.log(b64);
-			Tesseract.recognize(
-				b64,
-				'eng',
-				{ logger: m => console.log(m) },
-			).then(({ data: { text } }) => {
-				console.log(text);
+			os.api('drive/files/caption-image', {
+				url: img.src,
+			}).then(text => {
+				document.getElementById('recognized-text').innerText = text;
+				const allowedLength = 512 - this.inputValue.length;
+				this.inputValue += text.slice(0, allowedLength);
 			});
-			// await worker.load();
-			// await worker.loadLanguage('eng');
-			// await worker.initialize('eng');
-			// const { data: { text } } = await worker.recognize(img);
-			// console.log(text);
-			// // document.getElementById('recognized-text').innerText = text;
-			// // const allowedLength = 512 - this.inputValue.length;
-			// // this.inputValue += text.slice(0, allowedLength);
-			// await worker.terminate();
 		},
 	},
 });

packages/client/src/pages/follow.vue

@@ -20,7 +20,7 @@ async function follow(user): Promise<void> {
 		window.close();
 		return;
 	}
-	
+
 	os.apiWithDialog('following/create', {
 		userId: user.id,
 	});

yarn.lock

@@ -3084,6 +3084,7 @@ __metadata:
     summaly: 2.7.0
     syslog-pro: 1.0.0
     systeminformation: 5.12.11
+    tesseract.js: ^3.0.3
     tinycolor2: 1.4.2
     tmp: 0.2.1
     ts-loader: 9.4.1
@@ -4037,7 +4038,6 @@ __metadata:
     stringz: 2.1.0
     swiper: ^8.4.4
     syuilo-password-strength: 0.0.1
-    tesseract.js: ^3.0.3
     textarea-caret: 3.1.0
     three: 0.144.0
     throttle-debounce: 5.0.0