lq1405
f4d042f699
1. 移除软件包自带的本地 whisper（需单独安装） 2. 重构版本底层依赖，移除外部依赖 3. 修复 首页 暗黑模式不兼容的问题 4. 修复 SD 合并提示词报错
719 lines
27 KiB
TypeScript
719 lines
27 KiB
TypeScript
import { isEmpty } from 'lodash'
|
||
import { errorMessage, successMessage } from '../../Public/generalTools'
|
||
import { FfmpegOptions } from '../ffmpegOptions'
|
||
import { SubtitleSavePositionType } from '../../../define/enum/waterMarkAndSubtitle'
|
||
import { define } from '../../../define/define'
|
||
import path from 'path'
|
||
import {
|
||
CheckFileOrDirExist,
|
||
CheckFolderExistsOrCreate,
|
||
DeleteFolderAllFile,
|
||
GetFilesWithExtensions
|
||
} from '../../../define/Tools/file'
|
||
import { shell } from 'electron'
|
||
import { Book } from '../../../model/book'
|
||
import fs from 'fs'
|
||
import { GeneralResponse } from '../../../model/generalResponse'
|
||
import { BookServiceBasic } from '../ServiceBasic/bookServiceBasic'
|
||
import { LoggerStatus, OtherData, ResponseMessageType } from '../../../define/enum/softwareEnum'
|
||
import { LogScheduler } from '../task/logScheduler'
|
||
import { SubtitleModel } from '../../../model/subtitle'
|
||
import { BookTaskStatus, OperateBookType } from '../../../define/enum/bookEnum'
|
||
import axios from 'axios'
|
||
import { GptService } from '../GPT/gpt'
|
||
import FormData from 'form-data'
|
||
import { RetryWithBackoff } from '../../../define/Tools/common'
|
||
import { DEFINE_STRING } from '../../../define/define_string'
|
||
import { DraftTimeLineJson } from '../jianying/jianyingService'
|
||
const util = require('util')
|
||
const { spawn, exec } = require('child_process')
|
||
const execAsync = util.promisify(exec)
|
||
const fspromises = fs.promises
|
||
|
||
|
||
/**
|
||
* 去除水印和获取字幕相关操作
|
||
*/
|
||
export class Subtitle {
|
||
ffmpegOptions: FfmpegOptions
|
||
bookServiceBasic: BookServiceBasic
|
||
logScheduler: LogScheduler
|
||
gptService: GptService
|
||
|
||
constructor() {
|
||
this.bookServiceBasic = new BookServiceBasic()
|
||
this.logScheduler = new LogScheduler()
|
||
this.ffmpegOptions = new FfmpegOptions()
|
||
this.gptService = new GptService()
|
||
}
|
||
|
||
//#region 通用方法
|
||
|
||
/**
|
||
* 拆分视频总帧数，每秒多少帧，平分视频总帧数，后截取
|
||
* @param {*} videoDurationMs 视频的总时长（毫秒）
|
||
* @param {*} framesPerSecond 每秒截取多少帧
|
||
* @returns
|
||
*/
|
||
GenerateFrameTimes(videoDurationMs: number, framesPerSecond: number): number[] {
|
||
// 直接使用视频总时长（毫秒），不进行向下取整
|
||
const videoDurationSec = videoDurationMs / 1000
|
||
|
||
// 计算总共需要抽取的帧数，考虑到视频时长可能不是完整秒数，使用 Math.ceil 来确保至少获取到最后一秒内的帧
|
||
const totalFrames = Math.ceil(videoDurationSec * framesPerSecond)
|
||
|
||
// 计算两帧之间的时间间隔（毫秒）
|
||
const interval = 1000 / framesPerSecond
|
||
|
||
// 生成对应的时间点数组
|
||
const frameTimes = []
|
||
for (let i = 0; i < totalFrames; i++) {
|
||
// 使用 Math.min 确保最后一个时间点不会超过视频总时长
|
||
let timePoint = Math.min(Math.round(interval * i), videoDurationMs)
|
||
frameTimes.push(timePoint)
|
||
}
|
||
return frameTimes
|
||
}
|
||
|
||
/**
|
||
* 通用的小说获取分案的返回方法
|
||
* @param content 获取的文案内容
|
||
* @param book 小说实体类
|
||
* @param bookTask 小说任务实体类
|
||
* @param bookTaskDetail 小说任务分镜实体类
|
||
*/
|
||
async GetSubtitleLoggerAndResponse(content: string, progress: GeneralResponse.ProgressResponse, book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetail: Book.SelectBookTaskDetail): Promise<void> {
|
||
// 修改数据
|
||
await this.bookServiceBasic.UpdateBookTaskDetail(bookTaskDetail.id, {
|
||
word: content,
|
||
afterGpt: content
|
||
});
|
||
|
||
// let res = await this.basicReverse.GetCopywritingFunc(book, item);
|
||
// 将当前的数据实时返回，前端进行修改
|
||
this.bookServiceBasic.sendReturnMessage({
|
||
code: 1,
|
||
id: bookTaskDetail.id,
|
||
type: ResponseMessageType.GET_TEXT,
|
||
data: {
|
||
content: content,
|
||
progress: progress
|
||
} as GeneralResponse.SubtitleProgressResponse // 返回识别到的文案
|
||
}, DEFINE_STRING.BOOK.GET_COPYWRITING_RETURN)
|
||
|
||
// 添加日志
|
||
await this.logScheduler.AddLogToDB(
|
||
book.id,
|
||
book.type,
|
||
`${bookTaskDetail.name} 识别文案成功`,
|
||
bookTask.id,
|
||
LoggerStatus.SUCCESS
|
||
)
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region 获取字幕位置信息相关操作
|
||
|
||
/**
|
||
* 获取当前帧的文字信息
|
||
* @param {*} value 需要的参数的对象，必须包含以下参数
|
||
* @param {*} value.id 小说ID/小说分镜详细信息ID/null
|
||
* @param {*} value.type 保存的类型（主视频/分镜视频/后续会添加外部单独的视频提取）
|
||
*/
|
||
async GetCurrentFrameText(value: { id: any; type?: SubtitleSavePositionType; imageFolder: any }): Promise<GeneralResponse.ErrorItem | GeneralResponse.SuccessItem> {
|
||
try {
|
||
let iamgePaths = []
|
||
let imageFolder = value.imageFolder
|
||
? value.imageFolder
|
||
: path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`)
|
||
let imageFolderIsExist = await CheckFileOrDirExist(imageFolder)
|
||
if (!imageFolderIsExist) {
|
||
throw new Error('请先保存位置信息')
|
||
}
|
||
|
||
let images = await GetFilesWithExtensions(imageFolder, ['.png'])
|
||
let regex = /.*frame_.*\.png$/
|
||
images.forEach((element) => {
|
||
// 使用正则表达式测试文件名
|
||
if (regex.test(element)) {
|
||
iamgePaths.push(element)
|
||
}
|
||
})
|
||
|
||
// 开始识别
|
||
for (let i = 0; i < iamgePaths.length; i++) {
|
||
const imagePath = iamgePaths[i]
|
||
let scriptPath = path.join(define.scripts_path, 'LaiOcr/LaiOcr.exe')
|
||
let script = `cd "${path.dirname(scriptPath)}" && "${scriptPath}" "${imagePath}"`
|
||
let scriptRes = await execAsync(script, { maxBuffer: 1024 * 1024 * 10, encoding: 'utf-8' })
|
||
console.log(scriptRes)
|
||
if (scriptRes.error) {
|
||
throw new Error(scriptRes.error)
|
||
}
|
||
}
|
||
|
||
// 处理所有的图片完毕，遍历所有的数据返回
|
||
let textData = []
|
||
let jsonPath = await GetFilesWithExtensions(imageFolder, ['.json'])
|
||
for (let i = 0; i < jsonPath.length; i++) {
|
||
const element = jsonPath[i]
|
||
// 开始拼接
|
||
let texts = JSON.parse(await fspromises.readFile(element, 'utf-8'))
|
||
for (let j = 0; j < texts.length; j++) {
|
||
const text = texts[j][1][0]
|
||
textData.includes(text) ? null : textData.push(text)
|
||
}
|
||
}
|
||
return successMessage(
|
||
textData.join('\n'),
|
||
'获取当前帧的文字信息成功',
|
||
'WatermarkAndSubtitle_GetCurrentFrameText'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage(
|
||
'获取当前帧的文字信息失败，错误消息如下：' + error.toString(),
|
||
'WatermarkAndSubtitle_GetCurrentFrameText'
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 打开对应的ID的字幕提取的图片文件夹
|
||
* @param {*} value 需要的参数的对象，必须包含以下参数
|
||
* @param {*} value.id 小说ID/小说分镜详细信息ID/null
|
||
* @param {*} value.type 保存的类型（主视频/分镜视频/后续会添加外部单独的视频提取）
|
||
*/
|
||
async OpenBookSubtitlePositionScreenshot(value: { type: SubtitleSavePositionType; id: any }) {
|
||
try {
|
||
let folder
|
||
if (
|
||
value.type == SubtitleSavePositionType.MAIN_VIDEO ||
|
||
value.type == SubtitleSavePositionType.SETTING
|
||
) {
|
||
folder = path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`)
|
||
} else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) {
|
||
folder = path.join(define.project_path, `${value.id}/data/subtitle/${value.id}`)
|
||
}
|
||
|
||
// 判断文件夹是不是存在
|
||
let folderIsExist = await CheckFileOrDirExist(folder)
|
||
if (!folderIsExist) {
|
||
throw new Error('文件夹不存在，请先保存字幕位置信息')
|
||
}
|
||
|
||
// 打开文件夹
|
||
shell.openPath(folder)
|
||
return successMessage(
|
||
null,
|
||
'打开对应的文件夹成功',
|
||
'WatermarkAndSubtitle_OpenBookSubtitlePositionScreenshot'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage(
|
||
'打开字幕位置信息失败，错误消息如下：' + error.toString(),
|
||
'WatermarkAndSubtitle_OpenBookSubtitlePositionScreenshot'
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 保存反推的视频的文案位置信息（可以保存多个）
|
||
* @param {*} value 需要的参数的对象，必须包含以下参数
|
||
* @param {*} value.id 小说ID/小说分镜详细信息ID/null
|
||
* @param {*} value.bookSubtitlePosition 小说文案对应的位置
|
||
* @param {*} value.currentTime 视频当前保存的时间
|
||
* @param {*} value.type 保存的类型（主视频/分镜视频/后续会添加外部单独的视频提取）
|
||
* @returns
|
||
*/
|
||
async SaveBookSubtitlePosition(value: { type: SubtitleSavePositionType; id: string; bookSubtitlePosition: string | any[]; currentTime: number }) {
|
||
try {
|
||
let saveData = []
|
||
let videoPath
|
||
let outImagePath
|
||
// 小说视频保存
|
||
this.ffmpegOptions = new FfmpegOptions()
|
||
if (
|
||
value.type == SubtitleSavePositionType.MAIN_VIDEO ||
|
||
value.type == SubtitleSavePositionType.SETTING
|
||
) {
|
||
if (value.id == null) {
|
||
throw new Error('小说ID不能为空')
|
||
}
|
||
// 获取指定的小说
|
||
let bookRes = await this.bookServiceBasic.GetBookDataById(value.id)
|
||
if (bookRes == null) {
|
||
throw new Error('没有找到小说信息')
|
||
}
|
||
let book = bookRes
|
||
|
||
if (value.bookSubtitlePosition.length <= 0) {
|
||
throw new Error('没有获取到字幕信息')
|
||
}
|
||
|
||
videoPath = book.oldVideoPath
|
||
if (isEmpty(videoPath)) {
|
||
throw new Error('没有获取到视频路径')
|
||
}
|
||
outImagePath = path.join(book.bookFolderPath, `data/subtitle/${book.id}/frame.png`)
|
||
|
||
// 获取视频的宽高数据
|
||
let videoSizeRes = await this.ffmpegOptions.FfmpegGetVideoSize(videoPath)
|
||
if (videoSizeRes.code == 0) {
|
||
throw new Error(videoSizeRes.message)
|
||
}
|
||
let videoSize = videoSizeRes.data
|
||
|
||
// 开始计算比例
|
||
let videoWidth = videoSize.width
|
||
let videoHeight = videoSize.height
|
||
for (let i = 0; i < value.bookSubtitlePosition.length; i++) {
|
||
const element = value.bookSubtitlePosition[i]
|
||
let widthRate = videoWidth / element.videoWidth // 宽度比例
|
||
let heightRate = videoHeight / element.videoHeight // 高度比例
|
||
// 计算比例
|
||
let newStartX = widthRate * element.startX
|
||
let newStartY = heightRate * element.startY
|
||
let newWidth = widthRate * element.width
|
||
let newHeight = heightRate * element.height
|
||
saveData.push({
|
||
startX: newStartX,
|
||
startY: newStartY,
|
||
width: newWidth,
|
||
height: newHeight,
|
||
videoWidth: videoWidth,
|
||
videoHeight: videoHeight
|
||
})
|
||
}
|
||
|
||
// 数据保存
|
||
let saveRes = await this.bookServiceBasic.UpdateBookData(value.id, {
|
||
subtitlePosition: JSON.stringify(saveData)
|
||
})
|
||
} else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) {
|
||
// 小说分镜详细信息保存
|
||
if (value.id == null) {
|
||
throw new Error('小说分镜详细信息ID不能为空')
|
||
}
|
||
// 获取指定的小说分镜详细信息
|
||
let bookStoryboard = await this.bookServiceBasic.GetBookTaskDetailDataById(value.id)
|
||
if (bookStoryboard == null) {
|
||
throw new Error('没有找到小说分镜信息')
|
||
}
|
||
|
||
if (value.bookSubtitlePosition.length <= 0) {
|
||
throw new Error('没有获取到字幕信息')
|
||
}
|
||
|
||
videoPath = bookStoryboard.videoPath
|
||
if (isEmpty(videoPath)) {
|
||
throw new Error('没有获取到视频路径')
|
||
}
|
||
|
||
outImagePath = path.join(
|
||
define.project_path,
|
||
`${bookStoryboard.bookId}/data/subtitle/${bookStoryboard.id}/frame.png`
|
||
)
|
||
|
||
// 获取视频的宽高数据
|
||
this.ffmpegOptions = new FfmpegOptions()
|
||
let videoSizeRes = await this.ffmpegOptions.FfmpegGetVideoSize(videoPath)
|
||
if (videoSizeRes.code == 0) {
|
||
throw new Error(videoSizeRes.message)
|
||
}
|
||
let videoSize = videoSizeRes.data
|
||
|
||
// 开始计算比例
|
||
let videoWidth = videoSize.width
|
||
let videoHeight = videoSize.height
|
||
for (let i = 0; i < value.bookSubtitlePosition.length; i++) {
|
||
const element = value.bookSubtitlePosition[i]
|
||
let widthRate = videoWidth / element.videoWidth // 宽度比例
|
||
let heightRate = videoHeight / element.videoHeight // 高度比例
|
||
// 计算比例
|
||
let newStartX = widthRate * element.startX
|
||
let newStartY = heightRate * element.startY
|
||
let newWidth = widthRate * element.width
|
||
let newHeight = heightRate * element.height
|
||
saveData.push({
|
||
startX: newStartX,
|
||
startY: newStartY,
|
||
width: newWidth,
|
||
height: newHeight,
|
||
videoWidth: videoWidth,
|
||
videoHeight: videoHeight
|
||
})
|
||
}
|
||
// 数据保存
|
||
let saveRes = this.bookServiceBasic.UpdateBookTaskDetail(bookStoryboard.id, {
|
||
subtitlePosition: JSON.stringify(saveData)
|
||
})
|
||
}
|
||
|
||
// 开始设置裁剪出来的图片位置
|
||
// 裁剪一个示例图片
|
||
let saveImagePath = await this.ffmpegOptions.FfmpegGetVideoFramdAndClip(
|
||
videoPath,
|
||
value.currentTime * 1000,
|
||
outImagePath,
|
||
saveData
|
||
)
|
||
if (saveImagePath.code == 0) {
|
||
throw new Error(saveImagePath.message)
|
||
}
|
||
return successMessage(
|
||
saveImagePath.data,
|
||
'保存字幕位置信息成功',
|
||
'WatermarkAndSubtitle_SaveBookSubtitlePosition'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage(
|
||
'保存字幕位置信息失败，错误消息如下：' + error.toString(),
|
||
'WatermarkAndSubtitle_SaveBookSubtitlePosition'
|
||
)
|
||
}
|
||
}
|
||
//#endregion
|
||
|
||
//#region 本地OCR识别字幕相关操作
|
||
|
||
/**
|
||
* 获取当前视频中所有的字幕信息
|
||
* @param {*} value 需要的参数的对象，包含下面的参数
|
||
* @param {*} value.id 小说ID/小说分镜详细信息ID/null
|
||
* @param {*} value.type 保存的类型（主视频/分镜视频/后续会添加外部单独的视频提取）
|
||
* @param {*} value.videoPath 视频路径
|
||
* @param {*} value.subtitlePosition 字幕位置信息
|
||
*/
|
||
async GetVideoFrameText(value: Book.GetVideoFrameTextParams): Promise<GeneralResponse.ErrorItem | GeneralResponse.SuccessItem> {
|
||
try {
|
||
let videoPath = undefined
|
||
let tempImageFolder = undefined
|
||
let position = undefined
|
||
if (value.type == SubtitleSavePositionType.MAIN_VIDEO) {
|
||
let bookRes = await this.bookServiceBasic.GetBookDataById(value.id)
|
||
if (bookRes == null) {
|
||
throw new Error('没有找到小说对应的的视频地址')
|
||
}
|
||
let book = bookRes
|
||
tempImageFolder = path.join(define.project_path, `${book.id}/data/subtitle/${book.id}/temp`)
|
||
if (isEmpty(book.subtitlePosition)) {
|
||
throw new Error('请先保存位置信息')
|
||
}
|
||
position = JSON.parse(book.subtitlePosition)
|
||
videoPath = book.oldVideoPath
|
||
} else if (value.type == SubtitleSavePositionType.STORYBOARD_VIDEO) {
|
||
let bookTaskDetail = await this.bookServiceBasic.GetBookTaskDetailDataById(value.id);
|
||
if (bookTaskDetail == null) {
|
||
throw new Error("没有找到小说分镜详细信息")
|
||
}
|
||
|
||
tempImageFolder = path.join(define.project_path, `${bookTaskDetail.bookId}/data/subtitle/${bookTaskDetail.name}_${bookTaskDetail.id}/temp`)
|
||
if (isEmpty(value.subtitlePosition)) {
|
||
throw new Error('请先保存位置信息')
|
||
}
|
||
position = JSON.parse(value.subtitlePosition)
|
||
videoPath = bookTaskDetail.videoPath
|
||
} else {
|
||
throw new Error("不支持的操作");
|
||
}
|
||
|
||
await CheckFolderExistsOrCreate(tempImageFolder)
|
||
// 判断文件夹是不是存在，存在的话，将里面的所有文件删除
|
||
await DeleteFolderAllFile(tempImageFolder)
|
||
|
||
// 将视频进行抽帧，（目前是每秒1帧，时间小于一秒，抽一帧）
|
||
let getDurationRes = await this.ffmpegOptions.FfmpegGetVideoDuration(videoPath)
|
||
if (getDurationRes.code == 0) {
|
||
throw new Error(getDurationRes.message)
|
||
}
|
||
let videoDuration = getDurationRes.data
|
||
let frameTime = this.GenerateFrameTimes(videoDuration, 1)
|
||
for (let i = 0; i < frameTime.length; i++) {
|
||
const item = frameTime[i];
|
||
let name = i.toString().padStart(6, '0')
|
||
let imagePath = path.join(tempImageFolder, `frame_${name}.png`)
|
||
// 开始抽帧
|
||
let res = await this.ffmpegOptions.FfmpegGetVideoFramdAndClip(
|
||
videoPath,
|
||
item,
|
||
imagePath,
|
||
position
|
||
)
|
||
if (res.code == 0) {
|
||
throw new Error(res.message)
|
||
}
|
||
}
|
||
|
||
// 开始识别
|
||
let textRes = await this.GetCurrentFrameText({
|
||
id: value.id,
|
||
type: value.type,
|
||
imageFolder: tempImageFolder
|
||
})
|
||
|
||
let allTextData = [] as string[]
|
||
// 开始获取所有的数据
|
||
let jsonPaths = await GetFilesWithExtensions(tempImageFolder, ['.json'])
|
||
for (let i = 0; i < jsonPaths.length; i++) {
|
||
const element = jsonPaths[i]
|
||
// 开始拼接
|
||
let texts = JSON.parse(await fspromises.readFile(element, 'utf-8'))
|
||
for (let j = 0; j < texts.length; j++) {
|
||
const text = texts[j][1][0]
|
||
allTextData.includes(text) ? null : allTextData.push(text)
|
||
}
|
||
}
|
||
// 这边计算相似度，返回过于相似的数据
|
||
// let res = await RemoveSimilarTexts(allTextData)
|
||
|
||
return successMessage(
|
||
allTextData.join('，'),
|
||
'获取视频的的文案信息成功',
|
||
'WatermarkAndSubtitle_GetVideoFrameText'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage(
|
||
'提取视频的的文案信息失败，错误消息如下：' + error.toString(),
|
||
'WatermarkAndSubtitle_GetCurrentFrameText'
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 使用本地OCR识别字幕文案
|
||
* @param bookId 小说ID
|
||
* @param bookTaskId 小说任务ID
|
||
* @returns
|
||
*/
|
||
async GetCopywritingByLocalOcr(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetails: Book.SelectBookTaskDetail[]): Promise<GeneralResponse.ErrorItem | GeneralResponse.SuccessItem> {
|
||
try {
|
||
for (let i = 0; i < bookTaskDetails.length; i++) {
|
||
const item = bookTaskDetails[i];
|
||
let res = await this.GetVideoFrameText({
|
||
id: item.id,
|
||
videoPath: item.videoPath,
|
||
type: SubtitleSavePositionType.STORYBOARD_VIDEO,
|
||
subtitlePosition: book.subtitlePosition
|
||
})
|
||
if (res.code == 0) {
|
||
throw new Error(res.message)
|
||
}
|
||
// 修改数据，并返回
|
||
await this.GetSubtitleLoggerAndResponse(res.data, {
|
||
total: bookTaskDetails.length,
|
||
current: i + 1
|
||
}, book, bookTask, item)
|
||
}
|
||
return successMessage(null, "识别是所有文案成功", "Subtitle_GetCopywriting")
|
||
} catch (error) {
|
||
return errorMessage("获取分镜数据失败，失败信息如下：" + error.message, 'Subtitle_GetCopywriting')
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
|
||
//#region Lai_WHISPER识别字幕相关操作
|
||
|
||
/**
|
||
* 单个分离音频的方法
|
||
* @param book 小说数据
|
||
* @param bookTask 小说任务数据
|
||
* @param bookTaskDetail 小说任务详细信息数据
|
||
* @returns
|
||
*/
|
||
async SplitAudio(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetail: Book.SelectBookTaskDetail) {
|
||
// 开始分离音频
|
||
let videoPath = bookTaskDetail.videoPath
|
||
let audioPath = path.join(path.dirname(videoPath), bookTaskDetail.name + '.mp3');
|
||
let audioRes = await this.ffmpegOptions.FfmpegExtractAudio(bookTaskDetail.videoPath, audioPath)
|
||
if (audioRes.code == 0) {
|
||
let errorMessage = `分离音频失败，错误信息如下：${audioRes.message}`
|
||
await this.bookServiceBasic.UpdateBookTaskStatus(
|
||
bookTask.id,
|
||
BookTaskStatus.AUDIO_FAIL,
|
||
errorMessage
|
||
)
|
||
throw new Error(audioRes.message)
|
||
}
|
||
this.bookServiceBasic.UpdateBookTaskDetail(bookTaskDetail.id, {
|
||
audioPath: path.relative(define.project_path, audioPath)
|
||
})
|
||
|
||
// 推送成功消息
|
||
await this.logScheduler.AddLogToDB(
|
||
book.id,
|
||
book.type,
|
||
`${bookTaskDetail.name}分离音频成功，输出地址：${audioPath}`,
|
||
OtherData.DEFAULT,
|
||
LoggerStatus.SUCCESS
|
||
)
|
||
// 修改状态为分离音频成功
|
||
this.bookServiceBasic.UpdateBookTaskStatus(bookTask.id, BookTaskStatus.AUDIO_DONE)
|
||
return audioPath;
|
||
}
|
||
|
||
/**
|
||
* 使用LAI Whisper 进行文本识别，然后将繁体转换为简体
|
||
* @param audioPath 要识别的音频地址
|
||
* @param subtitleSetting 识别字幕设置
|
||
*/
|
||
async LaiWhisperApi(audioPath: string, subtitleSetting: SubtitleModel.subtitleSettingModel): Promise<string> {
|
||
// 开始调用LAI API识别
|
||
let formdata = new FormData()
|
||
formdata.append("file", fs.createReadStream(audioPath)); // 如果是Node.js环境，可以使用fs.createReadStream方法
|
||
formdata.append("model", "whisper-1");
|
||
formdata.append("response_format", "srt");
|
||
formdata.append("temperature", "0");
|
||
formdata.append("language", "zh");
|
||
formdata.append("prompt", isEmpty(subtitleSetting.laiWhisper.prompt) ? "eiusmod nulla" : subtitleSetting.laiWhisper.prompt);
|
||
let url = subtitleSetting.laiWhisper.url
|
||
if (!url.endsWith('/')) {
|
||
url = url + '/'
|
||
}
|
||
const config = {
|
||
method: 'post',
|
||
url: url + 'v1/audio/transcriptions',
|
||
headers: {
|
||
'Accept': 'application/json',
|
||
'Authorization': subtitleSetting.laiWhisper.apiKey,
|
||
'Content-Type': 'multipart/form-data',
|
||
...formdata.getHeaders() // 在Node.js环境中需要添加这一行
|
||
},
|
||
data: formdata
|
||
};
|
||
|
||
// laiwhisper 要做重试机制
|
||
let res = await RetryWithBackoff(async () => {
|
||
return await axios(config)
|
||
}, 5, 2000)
|
||
let text = res.data.text;
|
||
// 但是这边是繁体，需要转化为简体
|
||
// 请求也要做重试
|
||
let simpleText = await RetryWithBackoff(async () => {
|
||
return await this.gptService.ChineseTraditionalToSimplified(text, subtitleSetting.laiWhisper.apiKey, url);
|
||
}, 5, 2000);
|
||
|
||
console.log(res.data)
|
||
return simpleText;
|
||
}
|
||
|
||
/**
|
||
* 使用LAI Whisper识别字幕
|
||
* @param bookId 小说ID
|
||
* @param bookTaskId 小说任务ID
|
||
* @param subtitleSetting 提取文案相关设置
|
||
* @returns
|
||
*/
|
||
async GetCopywritingByLaiWhisper(book: Book.SelectBook, bookTask: Book.SelectBookTask, bookTaskDetails: Book.SelectBookTaskDetail[], subtitleSetting: SubtitleModel.subtitleSettingModel): Promise<GeneralResponse.ErrorItem | GeneralResponse.SuccessItem> {
|
||
try {
|
||
|
||
let emptyVideoPaths = [] as string[]
|
||
for (let i = 0; i < bookTaskDetails.length; i++) {
|
||
const element = bookTaskDetails[i];
|
||
// 将所有的分镜视频音频分开
|
||
if (isEmpty(element.videoPath)) {
|
||
emptyVideoPaths.push(element.name)
|
||
}
|
||
}
|
||
if (emptyVideoPaths.length > 0) {
|
||
throw new Error(`以下分镜视频没有找到对应的视频路径：${emptyVideoPaths.join("，")} \n 请先计算分镜`)
|
||
}
|
||
// 拆分音频和视频
|
||
for (let i = 0; i < bookTaskDetails.length; i++) {
|
||
const bookTaskDetail = bookTaskDetails[i];
|
||
// 开始分离音频
|
||
let audioPath = await this.SplitAudio(book, bookTask, bookTaskDetail)
|
||
let fileExist = await CheckFileOrDirExist(audioPath)
|
||
if (!fileExist) {
|
||
throw new Error('没有找到对应的音频文件');
|
||
}
|
||
// 开始调用LAI API识别
|
||
let content = await this.LaiWhisperApi(audioPath, subtitleSetting);
|
||
// 向前端发送数据
|
||
await this.GetSubtitleLoggerAndResponse(content, {
|
||
total: bookTaskDetails.length,
|
||
current: i + 1
|
||
}, book, bookTask, bookTaskDetail)
|
||
}
|
||
return successMessage(
|
||
null,
|
||
`所有音频识别成功`,
|
||
'Subtitle_GetCopywritingByLaiWhisper'
|
||
)
|
||
} catch (error) {
|
||
return errorMessage("获取分镜数据失败，失败信息如下：" + error.message, 'Subtitle_GetCopywritingByLaiWhisper')
|
||
}
|
||
}
|
||
//#endregion
|
||
|
||
|
||
//#region 本地Whisper识别字幕相关操作
|
||
|
||
async GetTextByLocalWhisper(frameTimeList: DraftTimeLineJson[], outDir: string, mp3Dir: string, localWhisperPath?: string): Promise<void> {
|
||
try {
|
||
let localWhisperPathExePath = localWhisperPath
|
||
if (isEmpty(localWhisperPathExePath)) {
|
||
localWhisperPathExePath = path.join(define.scripts_path, 'localWhisper/local_whisper.exe')
|
||
}
|
||
return new Promise((resolve, reject) => {
|
||
let child = spawn(
|
||
localWhisperPathExePath,
|
||
['-ts', outDir, mp3Dir],
|
||
{ encoding: 'utf-8' }
|
||
);
|
||
child.on('error', (error) => {
|
||
console.log('error=', error)
|
||
this.logScheduler.ReturnLogger(errorMessage("使用localWhisper识别字幕失败输出，失败信息如下：" + error.message))
|
||
reject(new Error(error.message))
|
||
})
|
||
child.stdout.on('data', (data) => {
|
||
console.log(data.toString())
|
||
this.logScheduler.ReturnLogger(successMessage(data.toString(), "使用localWhisper识别字幕输出"))
|
||
})
|
||
child.stderr.on('data', (data) => {
|
||
console.log('stderr=', data.toString())
|
||
this.logScheduler.ReturnLogger(errorMessage("使用localWhisper识别字幕失败输出，失败信息如下：stderr = " + data.toString()))
|
||
reject(new Error(data.toString()))
|
||
})
|
||
|
||
child.on('close', async (data) => {
|
||
console.log('data=', data.toString())
|
||
this.logScheduler.ReturnLogger(successMessage(data.toString(), "使用localWhisper识别字幕完成"))
|
||
let textPath = path.join(outDir, '文案.txt')
|
||
if (!await CheckFileOrDirExist(textPath)) {
|
||
throw new Error('没有找到识别输出的文案文件')
|
||
}
|
||
let text = await fspromises.readFile(textPath, 'utf-8')
|
||
let textLines = text.split(/\r?\n/)
|
||
let lastLine = textLines[textLines.length - 1]
|
||
// 丢掉最后一行
|
||
textLines = textLines.slice(0, -1)
|
||
|
||
if (textLines.length != frameTimeList.length) {
|
||
throw new Error('分镜和识别文案数量不一致')
|
||
}
|
||
// 保存文案
|
||
for (let i = 0; i < textLines.length; i++) {
|
||
const element = textLines[i];
|
||
frameTimeList[i].text = element
|
||
}
|
||
// 写出
|
||
await fspromises.writeFile(path.join(global.config.project_path, '文案.txt'), textLines.join('\n'), 'utf-8')
|
||
if (data == 0) {
|
||
this.logScheduler.ReturnLogger(successMessage(null, "使用localWhisper识别字幕完成"))
|
||
} else {
|
||
this.logScheduler.ReturnLogger(errorMessage("使用localWhisper识别字幕失败，失败信息请查看日志"))
|
||
}
|
||
resolve();
|
||
})
|
||
})
|
||
|
||
} catch (error) {
|
||
this.logScheduler.ReturnLogger(errorMessage("使用localWhisper识别字幕失败，失败信息如下：" + error.message))
|
||
throw error
|
||
}
|
||
}
|
||
//#endregion
|
||
}
|