import { isEmpty } from 'lodash' import { errorMessage, successMessage } from '../../Public/generalTools' import { FfmpegOptions } from '../ffmpegOptions' import { SubtitleSavePositionType } from '../../../define/enum/waterMarkAndSubtitle' import { define } from '../../../define/define' import path from 'path' import { CheckFileOrDirExist, CheckFolderExistsOrCreate, DeleteFolderAllFile, GetFilesWithExtensions } from '../../../define/Tools/file' import { shell } from 'electron' import { Book } from '../../../model/book' import fs from 'fs' import { GeneralResponse } from '../../../model/generalResponse' import { BookServiceBasic } from '../ServiceBasic/bookServiceBasic' import { LoggerStatus, OtherData, ResponseMessageType } from '../../../define/enum/softwareEnum' import { LogScheduler } from '../task/logScheduler' import { SubtitleModel } from '../../../model/subtitle' import { BookTaskStatus, OperateBookType } from '../../../define/enum/bookEnum' import axios from 'axios' import { GptService } from '../GPT/gpt' import FormData from 'form-data' import { RetryWithBackoff } from '../../../define/Tools/common' import { DEFINE_STRING } from '../../../define/define_string' import { DraftTimeLineJson } from '../jianying/jianyingService' const util = require('util') const { spawn, exec } = require('child_process') const execAsync = util.promisify(exec) const fspromises = fs.promises /** * 去除水印和获取字幕相关操作 */ export class Subtitle { ffmpegOptions: FfmpegOptions bookServiceBasic: BookServiceBasic logScheduler: LogScheduler gptService: GptService constructor() { this.bookServiceBasic = new BookServiceBasic() this.logScheduler = new LogScheduler() this.ffmpegOptions = new FfmpegOptions() this.gptService = new GptService() } //#region 通用方法 /** * 拆分视频总帧数,每秒多少帧,平分视频总帧数,后截取 * @param {*} videoDurationMs 视频的总时长(毫秒) * @param {*} framesPerSecond 每秒截取多少帧 * @returns */ GenerateFrameTimes(videoDurationMs: number, framesPerSecond: number): number[] { // 直接使用视频总时长(毫秒),不进行向下取整 const videoDurationSec = videoDurationMs / 1000 // 计算总共需要抽取的帧数,考虑到视频时长可能不是完整秒数,使用 Math.ceil 来确保至少获取到最后一秒内的帧 const totalFrames = Math.ceil(videoDurationSec * framesPerSecond) // 计算两帧之间的时间间隔(毫秒) const interval = 1000 / framesPerSecond // 生成对应的时间点数组 const frameTimes = [] for (let i = 0; i < totalFrames; i++) { // 使用 Math.min 确保最后一个时间点不会超过视频总时长 let timePoint = Math.min(Math.round(interval * i), videoDurationMs) frameTimes.push(timePoint) } return frameTimes } /** * 通用的小说获取分案的返回方法 * @param content 获取的文案内容 * @param book 小说实体类 * @param bookTask 小说任务实体类 * @param bookTaskDetail 小说任务分镜实体类 */ async GetSubtitleLoggerAndResponse(content: string, progress: GeneralResponse.ProgressResponse, book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetail: Book.SelectBookTaskDetail): Promise { // 修改数据 await this.bookServiceBasic.UpdateBookTaskDetail(bookTaskDetail.id, { word: content, afterGpt: content }); // let res = await this.basicReverse.GetCopywritingFunc(book, item); // 将当前的数据实时返回,前端进行修改 this.bookServiceBasic.sendReturnMessage({ code: 1, id: bookTaskDetail.id, type: ResponseMessageType.GET_TEXT, data: { content: content, progress: progress } as GeneralResponse.SubtitleProgressResponse // 返回识别到的文案 }, DEFINE_STRING.BOOK.GET_COPYWRITING_RETURN) // 添加日志 await this.logScheduler.AddLogToDB( book.id, book.type, `${bookTaskDetail.name} 识别文案成功`, bookTask.id, LoggerStatus.SUCCESS ) } //#endregion //#region 获取字幕位置信息相关操作 /** * 获取当前帧的文字信息 * @param {*} value 需要的参数的对象,必须包含以下参数 * @param {*} value.id 小说ID/小说分镜详细信息ID/null * @param {*} value.type 保存的类型(主视频/分镜视频/后续会添加外部单独的视频提取) */ async GetCurrentFrameText(value: { id: any; type?: SubtitleSavePositionType; imageFolder: any }): Promise { try { let iamgePaths = [] let imageFolder = value.imageFolder ? value.imageFolder : path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`) let imageFolderIsExist = await CheckFileOrDirExist(imageFolder) if (!imageFolderIsExist) { throw new Error('请先保存位置信息') } let images = await GetFilesWithExtensions(imageFolder, ['.png']) let regex = /.*frame_.*\.png$/ images.forEach((element) => { // 使用正则表达式测试文件名 if (regex.test(element)) { iamgePaths.push(element) } }) // 开始识别 for (let i = 0; i < iamgePaths.length; i++) { const imagePath = iamgePaths[i] let scriptPath = path.join(define.scripts_path, 'LaiOcr/LaiOcr.exe') let script = `cd "${path.dirname(scriptPath)}" && "${scriptPath}" "${imagePath}"` let scriptRes = await execAsync(script, { maxBuffer: 1024 * 1024 * 10, encoding: 'utf-8' }) console.log(scriptRes) if (scriptRes.error) { throw new Error(scriptRes.error) } } // 处理所有的图片完毕,遍历所有的数据返回 let textData = [] let jsonPath = await GetFilesWithExtensions(imageFolder, ['.json']) for (let i = 0; i < jsonPath.length; i++) { const element = jsonPath[i] // 开始拼接 let texts = JSON.parse(await fspromises.readFile(element, 'utf-8')) for (let j = 0; j < texts.length; j++) { const text = texts[j][1][0] textData.includes(text) ? null : textData.push(text) } } return successMessage( textData.join('\n'), '获取当前帧的文字信息成功', 'WatermarkAndSubtitle_GetCurrentFrameText' ) } catch (error) { return errorMessage( '获取当前帧的文字信息失败,错误消息如下:' + error.toString(), 'WatermarkAndSubtitle_GetCurrentFrameText' ) } } /** * 打开对应的ID的字幕提取的图片文件夹 * @param {*} value 需要的参数的对象,必须包含以下参数 * @param {*} value.id 小说ID/小说分镜详细信息ID/null * @param {*} value.type 保存的类型(主视频/分镜视频/后续会添加外部单独的视频提取) */ async OpenBookSubtitlePositionScreenshot(value: { type: SubtitleSavePositionType; id: any }) { try { let folder if ( value.type == SubtitleSavePositionType.MAIN_VIDEO || value.type == SubtitleSavePositionType.SETTING ) { folder = path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`) } else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) { folder = path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`) } // 判断文件夹是不是存在 let folderIsExist = await CheckFileOrDirExist(folder) if (!folderIsExist) { throw new Error('文件夹不存在,请先保存字幕位置信息') } // 打开文件夹 shell.openPath(folder) return successMessage( null, '打开对应的文件夹成功', 'WatermarkAndSubtitle_OpenBookSubtitlePositionScreenshot' ) } catch (error) { return errorMessage( '打开字幕位置信息失败,错误消息如下:' + error.toString(), 'WatermarkAndSubtitle_OpenBookSubtitlePositionScreenshot' ) } } /** * 保存反推的视频的文案位置信息(可以保存多个) * @param {*} value 需要的参数的对象,必须包含以下参数 * @param {*} value.id 小说ID/小说分镜详细信息ID/null * @param {*} value.bookSubtitlePosition 小说文案对应的位置 * @param {*} value.currentTime 视频当前保存的时间 * @param {*} value.type 保存的类型(主视频/分镜视频/后续会添加外部单独的视频提取) * @returns */ async SaveBookSubtitlePosition(value: { type: SubtitleSavePositionType; id: string; bookSubtitlePosition: string | any[]; currentTime: number }) { try { let saveData = [] let videoPath let outImagePath // 小说视频保存 this.ffmpegOptions = new FfmpegOptions() if ( value.type == SubtitleSavePositionType.MAIN_VIDEO || value.type == SubtitleSavePositionType.SETTING ) { if (value.id == null) { throw new Error('小说ID不能为空') } // 获取指定的小说 let bookRes = await this.bookServiceBasic.GetBookDataById(value.id) if (bookRes == null) { throw new Error('没有找到小说信息') } let book = bookRes if (value.bookSubtitlePosition.length <= 0) { throw new Error('没有获取到字幕信息') } videoPath = book.oldVideoPath if (isEmpty(videoPath)) { throw new Error('没有获取到视频路径') } outImagePath = path.join(book.bookFolderPath, `data/subtitle/${book.id}/frame.png`) // 获取视频的宽高数据 let videoSizeRes = await this.ffmpegOptions.FfmpegGetVideoSize(videoPath) if (videoSizeRes.code == 0) { throw new Error(videoSizeRes.message) } let videoSize = videoSizeRes.data // 开始计算比例 let videoWidth = videoSize.width let videoHeight = videoSize.height for (let i = 0; i < value.bookSubtitlePosition.length; i++) { const element = value.bookSubtitlePosition[i] let widthRate = videoWidth / element.videoWidth // 宽度比例 let heightRate = videoHeight / element.videoHeight // 高度比例 // 计算比例 let newStartX = widthRate * element.startX let newStartY = heightRate * element.startY let newWidth = widthRate * element.width let newHeight = heightRate * element.height saveData.push({ startX: newStartX, startY: newStartY, width: newWidth, height: newHeight, videoWidth: videoWidth, videoHeight: videoHeight }) } // 数据保存 let saveRes = await this.bookServiceBasic.UpdateBookData(value.id, { subtitlePosition: JSON.stringify(saveData) }) } else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) { // 小说分镜详细信息保存 if (value.id == null) { throw new Error('小说分镜详细信息ID不能为空') } // 获取指定的小说分镜详细信息 let bookStoryboard = await this.bookServiceBasic.GetBookTaskDetailDataById(value.id) if (bookStoryboard == null) { throw new Error('没有找到小说分镜信息') } if (value.bookSubtitlePosition.length <= 0) { throw new Error('没有获取到字幕信息') } videoPath = bookStoryboard.videoPath if (isEmpty(videoPath)) { throw new Error('没有获取到视频路径') } outImagePath = path.join( define.project_path, `${bookStoryboard.bookId}/data/subtitle/${bookStoryboard.id}/frame.png` ) // 获取视频的宽高数据 this.ffmpegOptions = new FfmpegOptions() let videoSizeRes = await this.ffmpegOptions.FfmpegGetVideoSize(videoPath) if (videoSizeRes.code == 0) { throw new Error(videoSizeRes.message) } let videoSize = videoSizeRes.data // 开始计算比例 let videoWidth = videoSize.width let videoHeight = videoSize.height for (let i = 0; i < value.bookSubtitlePosition.length; i++) { const element = value.bookSubtitlePosition[i] let widthRate = videoWidth / element.videoWidth // 宽度比例 let heightRate = videoHeight / element.videoHeight // 高度比例 // 计算比例 let newStartX = widthRate * element.startX let newStartY = heightRate * element.startY let newWidth = widthRate * element.width let newHeight = heightRate * element.height saveData.push({ startX: newStartX, startY: newStartY, width: newWidth, height: newHeight, videoWidth: videoWidth, videoHeight: videoHeight }) } // 数据保存 let saveRes = this.bookServiceBasic.UpdateBookTaskDetail(bookStoryboard.id, { subtitlePosition: JSON.stringify(saveData) }) } // 开始设置裁剪出来的图片位置 // 裁剪一个示例图片 let saveImagePath = await this.ffmpegOptions.FfmpegGetVideoFramdAndClip( videoPath, value.currentTime * 1000, outImagePath, saveData ) if (saveImagePath.code == 0) { throw new Error(saveImagePath.message) } return successMessage( saveImagePath.data, '保存字幕位置信息成功', 'WatermarkAndSubtitle_SaveBookSubtitlePosition' ) } catch (error) { return errorMessage( '保存字幕位置信息失败,错误消息如下:' + error.toString(), 'WatermarkAndSubtitle_SaveBookSubtitlePosition' ) } } //#endregion //#region 本地OCR识别字幕相关操作 /** * 获取当前视频中所有的字幕信息 * @param {*} value 需要的参数的对象,包含下面的参数 * @param {*} value.id 小说ID/小说分镜详细信息ID/null * @param {*} value.type 保存的类型(主视频/分镜视频/后续会添加外部单独的视频提取) * @param {*} value.videoPath 视频路径 * @param {*} value.subtitlePosition 字幕位置信息 */ async GetVideoFrameText(value: Book.GetVideoFrameTextParams): Promise { try { let videoPath = undefined let tempImageFolder = undefined let position = undefined if (value.type == SubtitleSavePositionType.MAIN_VIDEO) { let bookRes = await this.bookServiceBasic.GetBookDataById(value.id) if (bookRes == null) { throw new Error('没有找到小说对应的的视频地址') } let book = bookRes tempImageFolder = path.join(define.project_path, `${book.id}/data/subtitle/${book.id}/temp`) if (isEmpty(book.subtitlePosition)) { throw new Error('请先保存位置信息') } position = JSON.parse(book.subtitlePosition) videoPath = book.oldVideoPath } else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) { let bookTaskDetail = await this.bookServiceBasic.GetBookTaskDetailDataById(value.id); if (bookTaskDetail == null) { throw new Error("没有找到小说分镜详细信息") } tempImageFolder = path.join(define.project_path, `${bookTaskDetail.bookId}/data/subtitle/${bookTaskDetail.name}_${bookTaskDetail.id}/temp`) if (isEmpty(value.subtitlePosition)) { throw new Error('请先保存位置信息') } position = JSON.parse(value.subtitlePosition) videoPath = bookTaskDetail.videoPath } else { throw new Error("不支持的操作"); } await CheckFolderExistsOrCreate(tempImageFolder) // 判断文件夹是不是存在,存在的话,将里面的所有文件删除 await DeleteFolderAllFile(tempImageFolder) // 将视频进行抽帧,(目前是每秒1帧,时间小于一秒,抽一帧) let getDurationRes = await this.ffmpegOptions.FfmpegGetVideoDuration(videoPath) if (getDurationRes.code == 0) { throw new Error(getDurationRes.message) } let videoDuration = getDurationRes.data let frameTime = this.GenerateFrameTimes(videoDuration, 1) for (let i = 0; i < frameTime.length; i++) { const item = frameTime[i]; let name = i.toString().padStart(6, '0') let imagePath = path.join(tempImageFolder, `frame_${name}.png`) // 开始抽帧 let res = await this.ffmpegOptions.FfmpegGetVideoFramdAndClip( videoPath, item, imagePath, position ) if (res.code == 0) { throw new Error(res.message) } } // 开始识别 let textRes = await this.GetCurrentFrameText({ id: value.id, type: value.type, imageFolder: tempImageFolder }) let allTextData = [] as string[] // 开始获取所有的数据 let jsonPaths = await GetFilesWithExtensions(tempImageFolder, ['.json']) for (let i = 0; i < jsonPaths.length; i++) { const element = jsonPaths[i] // 开始拼接 let texts = JSON.parse(await fspromises.readFile(element, 'utf-8')) for (let j = 0; j < texts.length; j++) { const text = texts[j][1][0] allTextData.includes(text) ? null : allTextData.push(text) } } // 这边计算相似度,返回过于相似的数据 // let res = await RemoveSimilarTexts(allTextData) return successMessage( allTextData.join(','), '获取视频的的文案信息成功', 'WatermarkAndSubtitle_GetVideoFrameText' ) } catch (error) { return errorMessage( '提取视频的的文案信息失败,错误消息如下:' + error.toString(), 'WatermarkAndSubtitle_GetCurrentFrameText' ) } } /** * 使用本地OCR识别字幕文案 * @param bookId 小说ID * @param bookTaskId 小说任务ID * @returns */ async GetCopywritingByLocalOcr(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetails: Book.SelectBookTaskDetail[]): Promise { try { for (let i = 0; i < bookTaskDetails.length; i++) { const item = bookTaskDetails[i]; let res = await this.GetVideoFrameText({ id: item.id, videoPath: item.videoPath, type: SubtitleSavePositionType.STORYBOARD_VIDEO, subtitlePosition: book.subtitlePosition }) if (res.code == 0) { throw new Error(res.message) } // 修改数据,并返回 await this.GetSubtitleLoggerAndResponse(res.data, { total: bookTaskDetails.length, current: i + 1 }, book, bookTask, item) } return successMessage(null, "识别是所有文案成功", "Subtitle_GetCopywriting") } catch (error) { return errorMessage("获取分镜数据失败,失败信息如下:" + error.message, 'Subtitle_GetCopywriting') } } //#endregion //#region Lai_WHISPER识别字幕相关操作 /** * 单个分离音频的方法 * @param book 小说数据 * @param bookTask 小说任务数据 * @param bookTaskDetail 小说任务详细信息数据 * @returns */ async SplitAudio(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetail: Book.SelectBookTaskDetail) { // 开始分离音频 let videoPath = bookTaskDetail.videoPath let audioPath = path.join(path.dirname(videoPath), bookTaskDetail.name + '.mp3'); let audioRes = await this.ffmpegOptions.FfmpegExtractAudio(bookTaskDetail.videoPath, audioPath) if (audioRes.code == 0) { let errorMessage = `分离音频失败,错误信息如下:${audioRes.message}` await this.bookServiceBasic.UpdateBookTaskStatus( bookTask.id, BookTaskStatus.AUDIO_FAIL, errorMessage ) throw new Error(audioRes.message) } this.bookServiceBasic.UpdateBookTaskDetail(bookTaskDetail.id, { audioPath: path.relative(define.project_path, audioPath) }) // 推送成功消息 await this.logScheduler.AddLogToDB( book.id, book.type, `${bookTaskDetail.name}分离音频成功,输出地址:${audioPath}`, OtherData.DEFAULT, LoggerStatus.SUCCESS ) // 修改状态为分离音频成功 this.bookServiceBasic.UpdateBookTaskStatus(bookTask.id, BookTaskStatus.AUDIO_DONE) return audioPath; } /** * 使用LAI Whisper 进行文本识别,然后将繁体转换为简体 * @param audioPath 要识别的音频地址 * @param subtitleSetting 识别字幕设置 */ async LaiWhisperApi(audioPath: string, subtitleSetting: SubtitleModel.subtitleSettingModel): Promise { // 开始调用LAI API识别 let formdata = new FormData() formdata.append("file", fs.createReadStream(audioPath)); // 如果是Node.js环境,可以使用fs.createReadStream方法 formdata.append("model", "whisper-1"); formdata.append("response_format", "srt"); formdata.append("temperature", "0"); formdata.append("language", "zh"); formdata.append("prompt", isEmpty(subtitleSetting.laiWhisper.prompt) ? "eiusmod nulla" : subtitleSetting.laiWhisper.prompt); let url = subtitleSetting.laiWhisper.url if (!url.endsWith('/')) { url = url + '/' } const config = { method: 'post', url: url + 'v1/audio/transcriptions', headers: { 'Accept': 'application/json', 'Authorization': subtitleSetting.laiWhisper.apiKey, 'Content-Type': 'multipart/form-data', ...formdata.getHeaders() // 在Node.js环境中需要添加这一行 }, data: formdata }; // laiwhisper 要做重试机制 let res = await RetryWithBackoff(async () => { return await axios(config) }, 5, 2000) let text = res.data.text; // 但是这边是繁体,需要转化为简体 // 请求也要做重试 let simpleText = await RetryWithBackoff(async () => { return await this.gptService.ChineseTraditionalToSimplified(text, subtitleSetting.laiWhisper.apiKey, url); }, 5, 2000); console.log(res.data) return simpleText; } /** * 使用LAI Whisper识别字幕 * @param bookId 小说ID * @param bookTaskId 小说任务ID * @param subtitleSetting 提取文案相关设置 * @returns */ async GetCopywritingByLaiWhisper(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetails: Book.SelectBookTaskDetail[], subtitleSetting: SubtitleModel.subtitleSettingModel): Promise { try { let emptyVideoPaths = [] as string[] for (let i = 0; i < bookTaskDetails.length; i++) { const element = bookTaskDetails[i]; // 将所有的分镜视频音频分开 if (isEmpty(element.videoPath)) { emptyVideoPaths.push(element.name) } } if (emptyVideoPaths.length > 0) { throw new Error(`以下分镜视频没有找到对应的视频路径:${emptyVideoPaths.join(",")} \n 请先计算分镜`) } // 拆分音频和视频 for (let i = 0; i < bookTaskDetails.length; i++) { const bookTaskDetail = bookTaskDetails[i]; // 开始分离音频 let audioPath = await this.SplitAudio(book, bookTask, bookTaskDetail) let fileExist = await CheckFileOrDirExist(audioPath) if (!fileExist) { throw new Error('没有找到对应的音频文件'); } // 开始调用LAI API识别 let content = await this.LaiWhisperApi(audioPath, subtitleSetting); // 向前端发送数据 await this.GetSubtitleLoggerAndResponse(content, { total: bookTaskDetails.length, current: i + 1 }, book, bookTask, bookTaskDetail) } return successMessage( null, `所有音频识别成功`, 'Subtitle_GetCopywritingByLaiWhisper' ) } catch (error) { return errorMessage("获取分镜数据失败,失败信息如下:" + error.message, 'Subtitle_GetCopywritingByLaiWhisper') } } //#endregion //#region 本地Whisper识别字幕相关操作 async GetTextByLocalWhisper(frameTimeList: DraftTimeLineJson[], outDir: string, mp3Dir: string, localWhisperPath?: string): Promise { try { let localWhisperPathExePath = localWhisperPath if (isEmpty(localWhisperPathExePath)) { localWhisperPathExePath = path.join(define.scripts_path, 'localWhisper/local_whisper.exe') } return new Promise((resolve, reject) => { let child = spawn( localWhisperPathExePath, ['-ts', outDir, mp3Dir], { encoding: 'utf-8' } ); child.on('error', (error) => { console.log('error=', error) this.logScheduler.ReturnLogger(errorMessage("使用localWhisper识别字幕失败输出,失败信息如下:" + error.message)) reject(new Error(error.message)) }) child.stdout.on('data', (data) => { console.log(data.toString()) this.logScheduler.ReturnLogger(successMessage(data.toString(), "使用localWhisper识别字幕输出")) }) child.stderr.on('data', (data) => { console.log('stderr=', data.toString()) this.logScheduler.ReturnLogger(errorMessage("使用localWhisper识别字幕失败输出,失败信息如下:stderr = " + data.toString())) reject(new Error(data.toString())) }) child.on('close', async (data) => { console.log('data=', data.toString()) this.logScheduler.ReturnLogger(successMessage(data.toString(), "使用localWhisper识别字幕完成")) let textPath = path.join(outDir, '文案.txt') if (!await CheckFileOrDirExist(textPath)) { throw new Error('没有找到识别输出的文案文件') } let text = await fspromises.readFile(textPath, 'utf-8') let textLines = text.split(/\r?\n/) let lastLine = textLines[textLines.length - 1] // 丢掉最后一行 textLines = textLines.slice(0, -1) if (textLines.length != frameTimeList.length) { throw new Error('分镜和识别文案数量不一致') } // 保存文案 for (let i = 0; i < textLines.length; i++) { const element = textLines[i]; frameTimeList[i].text = element } // 写出 await fspromises.writeFile(path.join(global.config.project_path, '文案.txt'), textLines.join('\n'), 'utf-8') if (data == 0) { this.logScheduler.ReturnLogger(successMessage(null, "使用localWhisper识别字幕完成")) } else { this.logScheduler.ReturnLogger(errorMessage("使用localWhisper识别字幕失败,失败信息请查看日志")) } resolve(); }) }) } catch (error) { this.logScheduler.ReturnLogger(errorMessage("使用localWhisper识别字幕失败,失败信息如下:" + error.message)) throw error } } //#endregion }