SillyTavern 聊天系统分析

// 保存聊天（带完整性检查）
export async function trySaveChat(chatData, filePath, skipIntegrityCheck, handle, cardName, backupDirectory)

// 读取聊天数据
export function getChatData(chatFilePath)

// 获取聊天信息（用于列表展示）
export async function getChatInfo(pathToFile, additionalData, withMetadata)

// 检查聊天完整性
async function checkChatIntegrity(filePath, integritySlug)

import { sync as writeFileAtomicSync } from 'write-file-atomic';

// 原子写入：先写入临时文件，再重命名
writeFileAtomicSync(filePath, data, 'utf8');

const isBackupEnabled = !!getConfigValue('backups.chat.enabled', true, 'boolean');
const maxTotalChatBackups = Number(getConfigValue('backups.chat.maxTotalBackups', -1, 'number'));
const throttleInterval = Number(getConfigValue('backups.chat.throttleInterval', 10_000, 'number'));
const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'boolean');

const backupFunctions = new Map();

function getBackupFunction(handle) {
    if (!backupFunctions.has(handle)) {
        backupFunctions.set(handle, _.throttle(backupChat, throttleInterval, { 
            leading: true,  // 首次调用立即执行
            trailing: true  // 节流期结束后执行最后一次
        }));
    }
    return backupFunctions.get(handle);
}

async function checkChatIntegrity(filePath, integritySlug) {
    // 文件不存在时认为完整
    if (!fs.existsSync(filePath)) return true;
    
    // 读取首行获取完整性标识
    const firstLine = await readFirstLine(filePath);
    const jsonData = tryParse(firstLine);
    const chatIntegrity = jsonData?.chat_metadata?.integrity;
    
    // 无完整性元数据时跳过检查
    if (!chatIntegrity) return true;
    
    // 对比完整性标识
    return chatIntegrity === integritySlug;
}

const backupFile = path.join(directory, `${CHAT_BACKUPS_PREFIX}${name}_${generateTimestamp()}.jsonl`);
// 示例: chat_charactername_2024-01-01T12-00-00-000Z.jsonl

const chatFilePath = path.join(request.user.directories.groupChats, `${id}.jsonl`);

export async function migrateGroupChatsMetadataFormat(userDirectories)

function importOobaChat(userName, characterName, jsonData) {
    // data_visible: [[user_msg, char_msg], ...]
    for (const arr of jsonData.data_visible) {
        if (arr[0]) { /* 用户消息 */ }
        if (arr[1]) { /* 角色消息 */ }
    }
}

function importAgnaiChat(userName, characterName, jsonData) {
    // messages: [{userId?, msg}, ...]
    for (const message of jsonData.messages) {
        const isUser = !!message.userId;
        // msg -> mes
    }
}

function importCAIChat(userName, characterName, jsonData) {
    // histories.histories: [{msgs: [{src: {is_human}, text}]}, ...]
    for (const history of jsonData.histories.histories) {
        for (const msg of history.msgs) {
            // src.is_human -> is_user
            // text -> mes
        }
    }
}

function importKoboldLiteChat(userName, characterName, data) {
    const inputToken = '{{[INPUT]}}';
    const outputToken = '{{[OUTPUT]}}';
    
    // actions: [msg, ...]
    // prompt 作为首条消息
    // {{[INPUT]}} 标识用户消息
    // {{[OUTPUT]}} 标识角色消息
}

function importRisuChat(userName, characterName, jsonData) {
    // data.message: [{role, name, time, data}, ...]
    for (const message of jsonData.data.message) {
        const isUser = message.role === 'user';
        // data -> mes
        // time -> send_date (转换为 ISO 8601)
    }
}

function flattenChubChat(userName, characterName, lines) {
    // 处理嵌套的 mes.message 结构
    // 处理 swipes 数组中的嵌套消息
    function flattenSwipe(swipe) {
        return swipe.message ? swipe.message : swipe;
    }
}

if (request.body.format === 'jsonl') {
    const rawFile = fs.readFileSync(filename, 'utf8');
    return response.status(200).json({ result: rawFile });
}

rl.on('line', (line) => {
    const data = JSON.parse(line);
    if (data.is_system) return; // 跳过系统消息
    if (data.mes) {
        const name = data.name;
        const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n');
        buffer += `${name}: ${message}\n\n`;
    }
});

function calculateTotalGenTimeAndWordCount(chatDir, chat, uniqueGenStartTimes) {
    for (let line of lines) {
        const json = JSON.parse(line);
        
        // 计算生成时间
        if (json.gen_started && json.gen_finished) {
            totalGenTime += calculateGenTime(json.gen_started, json.gen_finished);
        }
        
        // 计算字数
        if (json.mes) {
            const wordCount = countWordsInString(json.mes);
            json.is_user ? userWordCount += wordCount : nonUserWordCount += wordCount;
            json.is_user ? userMsgCount++ : nonUserMsgCount++;
        }
        
        // 计算滑动次数
        if (json.swipes && json.swipes.length > 1) {
            totalSwipeCount += json.swipes.length - 1;
        }
    }
}

// 使用 sanitize-filename 防止路径遍历
import sanitize from 'sanitize-filename';
const safeFileName = sanitize(fileName);

// 使用 try-catch 包装文件操作
function tryReadFileSync(filePath) {
    try {
        return fs.readFileSync(filePath, 'utf8');
    } catch (error) {
        console.warn(`Failed to read file: ${filePath}`, error);
        return null;
    }
}

const fileStream = fs.createReadStream(pathToFile);
const rl = readline.createInterface({
    input: fileStream,
    crlfDelay: Infinity,
});

rl.on('line', (line) => {
    const jsonData = tryParse(line);
    // 处理每行数据
});

class IntegrityMismatchError extends Error {
    constructor(...params) {
        super(...params);
        if (Error.captureStackTrace) {
            Error.captureStackTrace(this, IntegrityMismatchError);
        }
        this.date = new Date();
    }
}