Unify chat timestamps format (#4806) * Unify chat timestamps format * Handle ISO timestamps in stats.js * Refactor timestamp parsing on server * Switch to ISO timestamps for character/messages creation dates * Fix type error * Early exist in saveGroupChat if group not found * Remove redundant fields from temp.chat export header * Auto-fix char creation date format on edit * Add name to fallback chat file names * Rename parseTimestamp server side function

5993084ee6ebd887625fa4a8404d21e56ab69f32

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: b9cea60c022351f99c20d231717112c52969863e
12 files changed, +171 -175Ignore whitespace
public/script.js+1 -4
@@ -1028,9 +1028,6 @@ function verifyCharactersSearchSortRule() {
10281028 }
10291029}
10301030
1031-/** @typedef {object} Character - A character */
1032-/** @typedef {object} Group - A group */
1033-
10341031/**
10351032 * @typedef {object} Entity - Object representing a display entity
10361033 * @property {Character|Group|import('./scripts/tags.js').Tag|*} item - The item
@@ -8248,7 +8245,7 @@ export function select_selected_character(chid, { switchMenu = true } = {}) {
82488245 $('#talkativeness_slider').val(characters[chid].talkativeness || talkativeness_default);
82498246 $('#mes_example_textarea').val(characters[chid].mes_example);
82508247 $('#selected_chat_pole').val(characters[chid].chat);
82518248 $('#create_date_pole').val(timestampToMoment(characters[chid].create_date).toISOString());
82528249 $('#avatar_url_pole').val(characters[chid].avatar);
82538250 $('#chat_import_avatar_url').val(characters[chid].avatar);
82548251 $('#chat_import_character_name').val(characters[chid].name);
public/scripts/RossAscends-mods.js+25 -31
@@ -162,43 +162,37 @@ export function shouldSendOnEnter() {
162162 }
163163}
164164
165-//RossAscends: Added function to format dates used in files and chat timestamps to a humanized format.
165+/**
166-//Mostly I wanted this to be for file names, but couldn't figure out exactly where the filename save code was as everything seemed to be connected.
166+ * Gets a humanized date time string from a given timestamp.
167-//Does not break old characters/chats, as the code just uses whatever timestamp exists in the chat.
167+ * @param {number} timestamp Timestamp in milliseconds
168-//New chats made with characters will use this new formatting.
168+ * @returns {string} Humanized date time string in the format `YYYY-MM-DD@HHhMMmSSsMSms`
169-export function humanizedDateTime() {
169+ */
170170 constexport nowfunction =humanizedDateTime(timestamp new= Date(Date.now()); {
171+ const date = new Date(timestamp);
171172 const dt = {
172- year: now.getFullYear(), month: now.getMonth() + 1, day: now.getDate(),
173+ year: date.getFullYear(),
173- hour: now.getHours(), minute: now.getMinutes(), second: now.getSeconds(),
174+ month: date.getMonth() + 1,
175+ day: date.getDate(),
176+ hour: date.getHours(),
177+ minute: date.getMinutes(),
178+ second: date.getSeconds(),
179+ millisecond: date.getMilliseconds(),
174180 };
175181 for (const key in dt) {
176- dt[key] = dt[key].toString().padStart(2, '0');
182+ const padLength = key === 'millisecond' ? 3 : 2;
183+ dt[key] = dt[key].toString().padStart(padLength, '0');
177184 }
178185 return `${dt.year}-${dt.month}-${dt.day}@${dt.hour}h${dt.minute}m${dt.second}s${dt.millisecond}ms`;
179186}
180187
181-//this is a common format version to display a timestamp on each chat message
188+/**
182-//returns something like: June 19, 2023 2:20pm
189+ * Gets a timestamp for messages in ISO 8601 format.
183-export function getMessageTimeStamp() {
190+ * @param {number} timestamp - optional timestamp in milliseconds
184- const date = Date.now();
191+ * @returns {string} ISO 8601 formatted timestamp
185- const months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];
192+ */
186- const d = new Date(date);
193+export function getMessageTimeStamp(timestamp = Date.now()) {
187194 const monthdate = months[d.getMonthnew Date(timestamp)];
188195 const day =return ddate.getDatetoISOString();
189- const year = d.getFullYear();
190- let hours = d.getHours();
191- const minutes = ('0' + d.getMinutes()).slice(-2);
192- let meridiem = 'am';
193- if (hours >= 12) {
194- meridiem = 'pm';
195- hours -= 12;
196- }
197- if (hours === 0) {
198- hours = 12;
199- }
200- const formattedDate = month + ' ' + day + ', ' + year + ' ' + hours + ':' + minutes + meridiem;
201- return formattedDate;
202196}
203197
204198
public/scripts/chats.js+2 -7
@@ -10,7 +10,6 @@ import {
1010 event_types,
1111 getCurrentChatId,
1212 getRequestHeaders,
13- name1,
1413 name2,
1514 reloadCurrentChat,
1615 saveSettingsDebounced,
@@ -2140,13 +2139,9 @@ export function initChatUtilities() {
21402139 await viewMessageFile(messageId, fileIndex);
21412140 });
21422141
21432142 $(document).on('click', '.assistant_note_export', async function (_e) {
21442143 const chatToSave = [
2145- {
2144+ { chat_metadata: chat_metadata },
2146- user_name: name1,
2147- character_name: name2,
2148- chat_metadata: chat_metadata,
2149- },
21502145 ...chat.filter(x => x?.extra?.type !== system_message_types.ASSISTANT_NOTE),
21512146 ];
21522147
public/scripts/extensions/stable-diffusion/index.js+1 -1
@@ -2865,7 +2865,7 @@ function getCharacterAvatarUrl() {
28652865 if (context.groupId) {
28662866 const groupMembers = context.groups.find(x => x.id === context.groupId)?.members;
28672867 const lastMessageAvatar = context.chat?.filter(x => !x.is_system && !x.is_user)?.slice(-1)[0]?.original_avatar;
28682868 const randomMemberAvatar = Array.isArray(groupMembers) ? groupMembers[Math.floor(Math.random() * groupMembers.length)]?.avatar : null;
28692869 const avatarToUse = lastMessageAvatar || randomMemberAvatar;
28702870 return formatCharacterAvatar(avatarToUse);
28712871 } else {
public/scripts/group-chats.js+7 -3
@@ -324,7 +324,7 @@ export async function getGroupChat(groupId, reload = false) {
324324 * Retrieves the members of a group
325325 *
326326 * @param {string} [groupId=selected_group] - The ID of the group to retrieve members from. Defaults to the currently selected group.
327327 * @returns {import('../script.js').Character[]} An array of character objects representing the members of the group. If the group is not found, an empty array is returned.
328328 */
329329export function getGroupMembers(groupId = selected_group) {
330330 const group = groups.find((x) => x.id === groupId);
@@ -617,7 +617,11 @@ function resetSelectedGroup() {
617617 */
618618async function saveGroupChat(groupId, shouldSaveGroup, force = false) {
619619 const group = groups.find(x => x.id == groupId);
620- const chat_id = group.chat_id;
620+ if (!group) {
621+ console.warn('Group not found', groupId);
622+ return;
623+ }
624+ const chatId = group.chat_id;
621625 group['date_last_chat'] = Date.now();
622626 /** @type {ChatHeader} */
623627 const chatHeader = {
@@ -626,7 +630,7 @@ async function saveGroupChat(groupId, shouldSaveGroup, force = false) {
626630 const response = await fetch('/api/chats/group/save', {
627631 method: 'POST',
628632 headers: getRequestHeaders(),
629633 body: JSON.stringify({ id: chat_idchatId, chat: [chatHeader, ...chat], force: force }),
630634 });
631635
632636 if (!response.ok) {
public/scripts/utils.js+2 -0
@@ -1105,6 +1105,8 @@ function parseTimestamp(timestamp) {
11051105 ms = typeof ms !== 'undefined' ? `.${ms.padStart(3, '0')}` : '';
11061106 return `${year.padStart(4, '0')}-${month.padStart(2, '0')}-${day.padStart(2, '0')}T${hour.padStart(2, '0')}:${min.padStart(2, '0')}:${sec.padStart(2, '0')}${ms}Z`;
11071107 };
1108+ // 2024-07-12@01h31m37s123ms
1109+ dtFmt.push({ callback: convertFromHumanized, pattern: /(\d{4})-(\d{1,2})-(\d{1,2})@(\d{1,2})h(\d{1,2})m(\d{1,2})s(\d{1,3})ms/ });
11081110 // 2024-7-12@01h31m37s
11091111 dtFmt.push({ callback: convertFromHumanized, pattern: /(\d{4})-(\d{1,2})-(\d{1,2})@(\d{1,2})h(\d{1,2})m(\d{1,2})s/ });
11101112 // 2024-6-5 @14h 56m 50s 682ms
src/byaf.js+3 -6
@@ -2,7 +2,7 @@ import { promises as fsPromises } from 'node:fs';
22import path from 'node:path';
33import urlJoin from 'url-join';
44import { DEFAULT_AVATAR_PATH } from './constants.js';
55import { extractFileFromZipBuffer, humanizedISO8601DateTime } from './util.js';
66
77/**
88 * A parser for BYAF (Backyard Archive Format) files.
@@ -267,7 +267,7 @@ export class ByafParser {
267267 extensions: { ...(character?.displayName && { 'display_name': character?.displayName }) }, // Preserve display name unmodified using extensions. "display_name" is not used by SillyTavern currently.
268268 },
269269 // @ts-ignore Non-standard spec extension
270270 create_date: humanizedISO8601DateTimenew Date().toISOString(),
271271 };
272272 }
273273 /**
@@ -330,13 +330,10 @@ export class ByafParser {
330330 * @returns {string} Chat data
331331 */
332332 static getChatFromScenario(scenario, userName, characterName, chatBackgrounds) {
333333 const chatStartDate = scenario?.messages?.length == 0 ? humanizedISO8601DateTimenew Date().toISOString() : scenario?.messages?.filter(m => 'createdAt' in m)[0].createdAt;
334334 const chatBackground = chatBackgrounds.find(bg => bg.paths.includes(scenario?.backgroundImage || ''))?.name || '';
335335 /** @type {object[]} */
336336 const chat = [{
337- user_name: userName,
338- character_name: characterName,
339- create_date: chatStartDate,
340337 chat_metadata: {
341338 scenario: scenario?.narrative ?? '',
342339 mes_example: ByafParser.formatExampleMessages(scenario?.exampleMessages),
src/endpoints/characters.js+18 -21
@@ -14,7 +14,7 @@ import storage from 'node-persist';
1414
1515import { AVATAR_WIDTH, AVATAR_HEIGHT, DEFAULT_AVATAR_PATH } from '../constants.js';
1616import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction } from '../middleware/validateFileName.js';
1717import { deepMerge, humanizedISO8601DateTimehumanizedDateTime, tryParse, extractFileFromZipBuffer, MemoryLimitedMap, getConfigValue, mutateJsonString, clientRelativePath, getUniqueName, sanitizeSafeCharacterReplacements } from '../util.js';
1818import { TavernCardValidator } from '../validator/TavernCardValidator.js';
1919import { parse, read, write } from '../character-card-parser.js';
2020import { readWorldInfoFile } from './worldinfo.js';
@@ -414,7 +414,7 @@ const processCharacter = async (item, directories, { shallow }) => {
414414 character['json_data'] = imgData;
415415 const charStat = fs.statSync(path.join(directories.characters, item));
416416 character['date_added'] = charStat.ctimeMs;
417417 character['create_date'] = jsonObject['create_date'] || humanizedISO8601DateTimenew Date(Math.round(charStat.ctimeMs)).toISOString();
418418 const chatsDirectory = path.join(directories.chats, item.replace('.png', ''));
419419
420420 const { chatSize, dateLastChat } = calculateChatSize(chatsDirectory);
@@ -452,7 +452,7 @@ function getCharaCardV2(jsonObject, directories, hoistDate = true) {
452452 jsonObject = convertToV2(jsonObject, directories);
453453
454454 if (hoistDate && !jsonObject.create_date) {
455455 jsonObject.create_date = humanizedISO8601DateTimenew Date().toISOString();
456456 }
457457 } else {
458458 jsonObject = readFromV2(jsonObject);
@@ -486,7 +486,7 @@ function convertToV2(char, directories) {
486486 depth_prompt_role: char.depth_prompt_role,
487487 }, directories);
488488
489489 result.chat = char.chat ?? humanizedISO8601DateTime`${char.name} - ${humanizedDateTime()}`;
490490 result.create_date = char.create_date;
491491
492492 return result;
@@ -551,7 +551,7 @@ function readFromV2(char) {
551551 char[charField] = v2Value;
552552 });
553553
554554 char['chat'] = char['chat'] ?? humanizedISO8601DateTime`${char.name} - ${humanizedDateTime()}`;
555555
556556 return char;
557557}
@@ -587,7 +587,7 @@ function charaFormatData(data, directories) {
587587 // Old ST extension fields (for backward compatibility, will be deprecated)
588588 _.set(char, 'creatorcomment', data.creator_notes || '');
589589 _.set(char, 'avatar', 'none');
590590 _.set(char, 'chat', data.ch_name + ' - ' + humanizedISO8601DateTimehumanizedDateTime());
591591 _.set(char, 'talkativeness', data.talkativeness || 0.5);
592592 _.set(char, 'fav', data.fav == 'true');
593593 _.set(char, 'tags', typeof data.tags == 'string' ? (data.tags.split(',').map(x => x.trim()).filter(x => x)) : data.tags || []);
@@ -624,9 +624,6 @@ function charaFormatData(data, directories) {
624624 _.set(char, 'data.extensions.depth_prompt.prompt', data.depth_prompt_prompt ?? '');
625625 _.set(char, 'data.extensions.depth_prompt.depth', depth_value);
626626 _.set(char, 'data.extensions.depth_prompt.role', role_value);
627- //_.set(char, 'data.extensions.create_date', humanizedISO8601DateTime());
628- //_.set(char, 'data.extensions.avatar', 'none');
629- //_.set(char, 'data.extensions.chat', data.ch_name + ' - ' + humanizedISO8601DateTime());
630627
631628 // V3 fields
632629 _.set(char, 'data.group_only_greetings', data.group_only_greetings ?? []);
@@ -746,8 +743,8 @@ async function importFromYaml(uploadPath, context, preservedFileName) {
746743 'name': yamlData.name,
747744 'description': yamlData.context ?? '',
748745 'first_mes': yamlData.greeting ?? '',
749746 'create_date': humanizedISO8601DateTimenew Date().toISOString(),
750747 'chat': `${yamlData.name} - ${humanizedISO8601DateTimehumanizedDateTime()}`,
751748 'personality': '',
752749 'creatorcomment': '',
753750 'avatar': 'none',
@@ -800,7 +797,7 @@ async function importFromCharX(uploadPath, { request }, preservedFileName) {
800797 }
801798
802799 unsetPrivateFields(card);
803800 card['create_date'] = humanizedISO8601DateTimenew Date().toISOString();
804801 card.name = sanitize(card.name);
805802 const fileName = preservedFileName || getPngName(card.name, request.user.directories);
806803 const result = await writeCharacterData(avatar, JSON.stringify(card), fileName, request);
@@ -822,7 +819,7 @@ async function importFromByaf(uploadPath, { request }, preservedFileName) {
822819 * @param {Partial<ByafScenario>} scenario
823820 */
824821 const createChatAsCurrentPersona = (scenario) => {
825822 const chatName = sanitize(`${scenario.title || card.name} - ${humanizedISO8601DateTimehumanizedDateTime()} imported.jsonl`, { replacement: sanitizeSafeCharacterReplacements });
826823 const filePath = path.join(request.user.directories.chats, path.basename(fileName), chatName);
827824 const dir = path.dirname(filePath);
828825 if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
@@ -898,7 +895,7 @@ async function importFromJson(uploadPath, { request }, preservedFileName) {
898895 importRisuSprites(request.user.directories, jsonData);
899896 unsetPrivateFields(jsonData);
900897 jsonData = readFromV2(jsonData);
901898 jsonData['create_date'] = humanizedISO8601DateTimenew Date().toISOString();
902899 const pngName = preservedFileName || getPngName(jsonData.data?.name || jsonData.name, request.user.directories);
903900 const char = JSON.stringify(jsonData);
904901 const result = await writeCharacterData(DEFAULT_AVATAR_PATH, char, pngName, request);
@@ -917,10 +914,10 @@ async function importFromJson(uploadPath, { request }, preservedFileName) {
917914 'personality': jsonData.personality ?? '',
918915 'first_mes': jsonData.first_mes ?? '',
919916 'avatar': 'none',
920917 'chat': jsonData.name + ' - ' + humanizedISO8601DateTimehumanizedDateTime(),
921918 'mes_example': jsonData.mes_example ?? '',
922919 'scenario': jsonData.scenario ?? '',
923920 'create_date': humanizedISO8601DateTimenew Date().toISOString(),
924921 'talkativeness': jsonData.talkativeness ?? 0.5,
925922 'creator': jsonData.creator ?? '',
926923 'tags': jsonData.tags ?? '',
@@ -943,10 +940,10 @@ async function importFromJson(uploadPath, { request }, preservedFileName) {
943940 'personality': '',
944941 'first_mes': jsonData.char_greeting ?? '',
945942 'avatar': 'none',
946943 'chat': jsonData.name + ' - ' + humanizedISO8601DateTimehumanizedDateTime(),
947944 'mes_example': jsonData.example_dialogue ?? '',
948945 'scenario': jsonData.world_scenario ?? '',
949946 'create_date': humanizedISO8601DateTimenew Date().toISOString(),
950947 'talkativeness': jsonData.talkativeness ?? 0.5,
951948 'creator': jsonData.creator ?? '',
952949 'tags': jsonData.tags ?? '',
@@ -981,7 +978,7 @@ async function importFromPng(uploadPath, { request }, preservedFileName) {
981978 importRisuSprites(request.user.directories, jsonData);
982979 unsetPrivateFields(jsonData);
983980 jsonData = readFromV2(jsonData);
984981 jsonData['create_date'] = humanizedISO8601DateTimenew Date().toISOString();
985982 const char = JSON.stringify(jsonData);
986983 const result = await writeCharacterData(uploadPath, char, pngName, request);
987984 fs.unlinkSync(uploadPath);
@@ -1000,10 +997,10 @@ async function importFromPng(uploadPath, { request }, preservedFileName) {
1000997 'personality': jsonData.personality ?? '',
1001998 'first_mes': jsonData.first_mes ?? '',
1002999 'avatar': 'none',
10031000 'chat': jsonData.name + ' - ' + humanizedISO8601DateTimehumanizedDateTime(),
10041001 'mes_example': jsonData.mes_example ?? '',
10051002 'scenario': jsonData.scenario ?? '',
10061003 'create_date': humanizedISO8601DateTimenew Date().toISOString(),
10071004 'talkativeness': jsonData.talkativeness ?? 0.5,
10081005 'creator': jsonData.creator ?? '',
10091006 'tags': jsonData.tags ?? '',
src/endpoints/chats.js+22 -24
@@ -11,7 +11,7 @@ import _ from 'lodash';
1111import validateAvatarUrlMiddleware from '../middleware/validateFileName.js';
1212import {
1313 getConfigValue,
1414 humanizedISO8601DateTimehumanizedDateTime,
1515 tryParse,
1616 generateTimestamp,
1717 removeOldBackups,
@@ -106,9 +106,7 @@ process.on('exit', () => {
106106function importOobaChat(userName, characterName, jsonData) {
107107 /** @type {object[]} */
108108 const chat = [{
109109 user_namechat_metadata: userName{},
110- character_name: characterName,
111- create_date: humanizedISO8601DateTime(),
112110 }];
113111
114112 for (const arr of jsonData.data_visible) {
@@ -116,8 +114,9 @@ function importOobaChat(userName, characterName, jsonData) {
116114 const userMessage = {
117115 name: userName,
118116 is_user: true,
119117 send_date: humanizedISO8601DateTimenew Date().toISOString(),
120118 mes: arr[0],
119+ extra: {},
121120 };
122121 chat.push(userMessage);
123122 }
@@ -125,8 +124,9 @@ function importOobaChat(userName, characterName, jsonData) {
125124 const charMessage = {
126125 name: characterName,
127126 is_user: false,
128127 send_date: humanizedISO8601DateTimenew Date().toISOString(),
129128 mes: arr[1],
129+ extra: {},
130130 };
131131 chat.push(charMessage);
132132 }
@@ -145,9 +145,7 @@ function importOobaChat(userName, characterName, jsonData) {
145145function importAgnaiChat(userName, characterName, jsonData) {
146146 /** @type {object[]} */
147147 const chat = [{
148148 user_namechat_metadata: userName{},
149- character_name: characterName,
150- create_date: humanizedISO8601DateTime(),
151149 }];
152150
153151 for (const message of jsonData.messages) {
@@ -155,8 +153,9 @@ function importAgnaiChat(userName, characterName, jsonData) {
155153 chat.push({
156154 name: isUser ? userName : characterName,
157155 is_user: isUser,
158156 send_date: humanizedISO8601DateTimenew Date().toISOString(),
159157 mes: message.msg,
158+ extra: {},
160159 });
161160 }
162161
@@ -178,16 +177,15 @@ function importCAIChat(userName, characterName, jsonData) {
178177 */
179178 function convert(history) {
180179 const starter = {
181180 user_namechat_metadata: userName{},
182- character_name: characterName,
183- create_date: humanizedISO8601DateTime(),
184181 };
185182
186183 const historyData = history.msgs.map((msg) => ({
187184 name: msg.src.is_human ? userName : characterName,
188185 is_user: msg.src.is_human,
189186 send_date: humanizedISO8601DateTimenew Date().toISOString(),
190187 mes: msg.text,
188+ extra: {},
191189 }));
192190
193191 return [starter, ...historyData];
@@ -215,7 +213,8 @@ function importKoboldLiteChat(_userName, _characterName, data) {
215213 name: isUser ? header.user_name : header.character_name,
216214 is_user: isUser,
217215 mes: msg.replaceAll(inputToken, '').replaceAll(outputToken, '').trim(),
218216 send_date: new Date().nowtoISOString(),
217+ extra: {},
219218 };
220219 }
221220
@@ -276,9 +275,7 @@ function flattenChubChat(userName, characterName, lines) {
276275function importRisuChat(userName, characterName, jsonData) {
277276 /** @type {object[]} */
278277 const chat = [{
279278 user_namechat_metadata: userName{},
280- character_name: characterName,
281- create_date: humanizedISO8601DateTime(),
282279 }];
283280
284281 for (const message of jsonData.data.message) {
@@ -286,8 +283,9 @@ function importRisuChat(userName, characterName, jsonData) {
286283 chat.push({
287284 name: message.name ?? (isUser ? userName : characterName),
288285 is_user: isUser,
289286 send_date: new Date(Number(message.time ?? Date.now())).toISOString(),
290287 mes: message.data ?? '',
288+ extra: {},
291289 });
292290 }
293291
@@ -417,7 +415,7 @@ export async function getChatInfo(pathToFile, additionalData = {}, withMetadata
417415 if (jsonData && (jsonData.name || jsonData.character_name || jsonData.chat_metadata)) {
418416 chatData.chat_items = (itemCounter - 1);
419417 chatData.mes = jsonData['mes'] || '[The message is empty]';
420418 chatData.last_mes = jsonData['send_date'] || new Date(Math.round(stats.mtimeMs)).toISOString();
421419
422420 res(chatData);
423421 } else {
@@ -623,7 +621,7 @@ router.post('/group/import', function (request, response) {
623621 return response.sendStatus(400);
624622 }
625623
626624 const chatname = humanizedISO8601DateTimehumanizedDateTime();
627625 const pathToUpload = path.join(filedata.destination, filedata.filename);
628626 const pathToNewFile = path.join(request.user.directories.groupChats, `${chatname}.jsonl`);
629627 fs.copyFileSync(pathToUpload, pathToNewFile);
@@ -675,7 +673,7 @@ router.post('/import', validateAvatarUrlMiddleware, function (request, response)
675673 }
676674
677675 const handleChat = (chat) => {
678676 const fileName = `${characterName} - ${humanizedISO8601DateTimehumanizedDateTime()} imported.jsonl`;
679677 const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
680678 fileNames.push(fileName);
681679 writeFileAtomicSync(filePath, chat, 'utf8');
@@ -714,7 +712,7 @@ router.post('/import', validateAvatarUrlMiddleware, function (request, response)
714712 console.warn('Failed to flatten Chub Chat data: ', error);
715713 }
716714
717715 const fileName = `${characterName} - ${humanizedISO8601DateTimehumanizedDateTime()} imported.jsonl`;
718716 const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
719717 fileNames.push(fileName);
720718 if (flattenedChat !== data) {
@@ -879,7 +877,7 @@ router.post('/search', validateAvatarUrlMiddleware, function (request, response)
879877 }
880878
881879 const lastMessage = messages[messages.length - 1];
882880 const lastMesDate = lastMessage?.send_date || Math.roundnew Date(fs.statSync(chatFile.path).mtimeMs).toISOString();
883881
884882 // If no search query, just return metadata
885883 if (!query) {
src/endpoints/groups.js+2 -2
@@ -6,7 +6,7 @@ import express from 'express';
66import sanitize from 'sanitize-filename';
77import { sync as writeFileAtomicSync, default as writeFileAtomic } from 'write-file-atomic';
88
99import { color, humanizedISO8601DateTime, tryParse } from '../util.js';
1010import { getFileNameValidationFunction } from '../middleware/validateFileName.js';
1111
1212export const router = express.Router();
@@ -127,7 +127,7 @@ router.post('/all', (request, response) => {
127127 const group = JSON.parse(fileContents);
128128 const groupStat = fs.statSync(filePath);
129129 group['date_added'] = groupStat.birthtimeMs;
130130 group['create_date'] = humanizedISO8601DateTimenew Date(groupStat.birthtimeMs).toISOString();
131131
132132 let chat_size = 0;
133133 let date_last_chat = 0;
src/endpoints/stats.js+67 -65
@@ -12,6 +12,21 @@ import { getAllUserHandles, getUserDirectories } from '../users.js';
1212
1313const STATS_FILE = 'stats.json';
1414
15+const monthNames = [
16+ 'January',
17+ 'February',
18+ 'March',
19+ 'April',
20+ 'May',
21+ 'June',
22+ 'July',
23+ 'August',
24+ 'September',
25+ 'October',
26+ 'November',
27+ 'December',
28+];
29+
1530/**
1631 * @type {Map<string, Object>} The stats object for each user.
1732 */
@@ -23,92 +38,79 @@ const TIMESTAMPS = new Map();
2338
2439/**
2540 * Convert a timestamp to an integer timestamp.
26- * (sorry, it's momentless for now, didn't want to add a package just for this)
2741 * This function can handle several different timestamp formats:
2842 * 1. UnixDate.now timestamps (the number of secondsmilliseconds since the Unix Epoch)
2943 * 2. ST "humanized" timestamps, formatted like "`YYYY-MM-DD @HHh MMm SSs ms"HHhMMmSSsMSms`
3044 * 3. Date strings in the format "`Month DD, YYYY H:MMam/pm"`
45+ * 4. ISO 8601 formatted strings
46+ * 5. Date objects
3147 *
3248 * The function returns the timestamp as the number of milliseconds since
3349 * the Unix Epoch, which can be converted to a JavaScript Date object with new Date().
3450 *
3551 * @param {string|number|Date} timestamp - The timestamp to convert.
3652 * @returns {number} The timestamp in milliseconds since the Unix Epoch, or 0 if the input cannot be parsed.
3753 *
3854 * @example
3955 * // Unix timestamp
4056 * timestampToMomentparseTimestamp(1609459200);
4157 * // ST humanized timestamp
4258 * timestampToMomentparseTimestamp("2021-01-01 \@00h 00m 00s 000ms");
4359 * // Date string
4460 * timestampToMomentparseTimestamp("January 1, 2021 12:00am");
4561 */
4662function timestampToMomentparseTimestamp(timestamp) {
4763 if (!timestamp) {
4864 return 0;
4965 }
5066
51- if (typeof timestamp === 'number') {
67+ // Date object
52- return timestamp;
68+ if (timestamp instanceof Date) {
69+ return timestamp.getTime();
5370 }
5471
55- const pattern1 =
72+ // Unix time
56- /(\d{4})-(\d{1,2})-(\d{1,2}) @(\d{1,2})h (\d{1,2})m (\d{1,2})s (\d{1,3})ms/;
73+ if (typeof timestamp === 'number' || /^\d+$/.test(timestamp)) {
5774 const replacement1unixTime = Number(timestamp);
58- match,
75+ const isValid = Number.isFinite(unixTime) && !Number.isNaN(unixTime) && unixTime >= 0;
59- year,
76+ if (!isValid) return 0;
60- month,
77+ return new Date(unixTime).getTime();
61- day,
78+ }
62- hour,
79+
63- minute,
80+ // ISO 8601 format
64- second,
81+ const isoPattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/;
65- millisecond,
82+ if (isoPattern.test(timestamp)) {
66- ) => {
83+ return new Date(timestamp).getTime();
67- return `${year}-${month.padStart(2, '0')}-${day.padStart(
68- 2,
69- '0',
70- )}T${hour.padStart(2, '0')}:${minute.padStart(
71- 2,
72- '0',
73- )}:${second.padStart(2, '0')}.${millisecond.padStart(3, '0')}Z`;
74- };
75- const isoTimestamp1 = timestamp.replace(pattern1, replacement1);
76- if (!isNaN(Number(new Date(isoTimestamp1)))) {
77- return new Date(isoTimestamp1).getTime();
7884 }
7985
80- const pattern2 = /(\w+)\s(\d{1,2}),\s(\d{4})\s(\d{1,2}):(\d{1,2})(am|pm)/i;
86+ let dateFormats = [];
81- const replacement2 = (match, month, day, year, hour, minute, meridiem) => {
87+
82- const monthNames = [
88+ // meridiem-based format
83- 'January',
89+ const convertFromMeridiemBased = (_, month, day, year, hour, minute, meridiem) => {
84- 'February',
85- 'March',
86- 'April',
87- 'May',
88- 'June',
89- 'July',
90- 'August',
91- 'September',
92- 'October',
93- 'November',
94- 'December',
95- ];
9690 const monthNum = monthNames.indexOf(month) + 1;
97- const hour24 =
91+ const hour24 = meridiem.toLowerCase() === 'pm' ? (parseInt(hour, 10) % 12) + 12 : parseInt(hour, 10) % 12;
98- meridiem.toLowerCase() === 'pm'
92+ return `${year}-${monthNum}-${day.padStart(2, '0')}T${hour24.toString().padStart(2, '0')}:${minute.padStart(2, '0')}:00`;
99- ? (parseInt(hour, 10) % 12) + 12
93+ };
100- : parseInt(hour, 10) % 12;
94+ // June 19, 2023 2:20pm
101- return `${year}-${monthNum.toString().padStart(2, '0')}-${day.padStart(
95+ dateFormats.push({ callback: convertFromMeridiemBased, pattern: /(\w+)\s(\d{1,2}),\s(\d{4})\s(\d{1,2}):(\d{1,2})(am|pm)/i });
102- 2,
96+
103- '0',
97+ // ST "humanized" format patterns
104- )}T${hour24.toString().padStart(2, '0')}:${minute.padStart(
98+ const convertFromHumanized = (_, year, month, day, hour, min, sec, ms) => {
105- 2,
99+ ms = typeof ms !== 'undefined' ? `.${ms.padStart(3, '0')}` : '';
106- '0',
100+ return `${year.padStart(4, '0')}-${month.padStart(2, '0')}-${day.padStart(2, '0')}T${hour.padStart(2, '0')}:${min.padStart(2, '0')}:${sec.padStart(2, '0')}${ms}Z`;
107- )}:00Z`;
108101 };
109- const isoTimestamp2 = timestamp.replace(pattern2, replacement2);
102+ // 2024-07-12@01h31m37s123ms
110- if (!isNaN(Number(new Date(isoTimestamp2)))) {
103+ dateFormats.push({ callback: convertFromHumanized, pattern: /(\d{4})-(\d{1,2})-(\d{1,2})@(\d{1,2})h(\d{1,2})m(\d{1,2})s(\d{1,3})ms/ });
111- return new Date(isoTimestamp2).getTime();
104+ // 2024-7-12@01h31m37s
105+ dateFormats.push({ callback: convertFromHumanized, pattern: /(\d{4})-(\d{1,2})-(\d{1,2})@(\d{1,2})h(\d{1,2})m(\d{1,2})s/ });
106+ // 2024-6-5 @14h 56m 50s 682ms
107+ dateFormats.push({ callback: convertFromHumanized, pattern: /(\d{4})-(\d{1,2})-(\d{1,2}) @(\d{1,2})h (\d{1,2})m (\d{1,2})s (\d{1,3})ms/ });
108+
109+ for (const x of dateFormats) {
110+ const rgxMatch = timestamp.match(x.pattern);
111+ if (!rgxMatch) continue;
112+ const isoTimestamp = x.callback(...rgxMatch);
113+ return new Date(isoTimestamp).getTime();
112114 }
113115
114116 return 0;
@@ -416,7 +418,7 @@ function calculateTotalGenTimeAndWordCount(
416418 // If this is the first user message, set the first chat time
417419 if (json.is_user) {
418420 //get min between firstChatTime and timestampToMoment(json.send_date)
419421 firstChatTime = Math.min(timestampToMomentparseTimestamp(json.send_date), firstChatTime);
420422 }
421423 } catch (error) {
422424 console.error(`Error parsing line ${line}: ${error}`);
src/util.js+21 -11
@@ -389,17 +389,27 @@ export function uuidv4() {
389389 });
390390}
391391
392-export function humanizedISO8601DateTime(date) {
392+/**
393- let baseDate = typeof date === 'number' ? new Date(date) : new Date();
393+ * Gets a humanized date time string from a given timestamp.
394- let humanYear = baseDate.getFullYear();
394+ * @param {number} timestamp Timestamp in milliseconds
395- let humanMonth = (baseDate.getMonth() + 1);
395+ * @returns {string} Humanized date time string in the format `YYYY-MM-DD@HHhMMmSSsMSms`
396- let humanDate = baseDate.getDate();
396+ */
397- let humanHour = (baseDate.getHours() < 10 ? '0' : '') + baseDate.getHours();
397+export function humanizedDateTime(timestamp = Date.now()) {
398- let humanMinute = (baseDate.getMinutes() < 10 ? '0' : '') + baseDate.getMinutes();
398+ const date = new Date(timestamp);
399- let humanSecond = (baseDate.getSeconds() < 10 ? '0' : '') + baseDate.getSeconds();
399+ const dt = {
400- let humanMillisecond = (baseDate.getMilliseconds() < 10 ? '0' : '') + baseDate.getMilliseconds();
400+ year: date.getFullYear(),
401- let HumanizedDateTime = (humanYear + '-' + humanMonth + '-' + humanDate + ' @' + humanHour + 'h ' + humanMinute + 'm ' + humanSecond + 's ' + humanMillisecond + 'ms');
401+ month: date.getMonth() + 1,
402- return HumanizedDateTime;
402+ day: date.getDate(),
403+ hour: date.getHours(),
404+ minute: date.getMinutes(),
405+ second: date.getSeconds(),
406+ millisecond: date.getMilliseconds(),
407+ };
408+ for (const key in dt) {
409+ const padLength = key === 'millisecond' ? 3 : 2;
410+ dt[key] = dt[key].toString().padStart(padLength, '0');
411+ }
412+ return `${dt.year}-${dt.month}-${dt.day}@${dt.hour}h${dt.minute}m${dt.second}s${dt.millisecond}ms`;
403413}
404414
405415export function tryParse(str) {