<?php

class TWXChatUtil
{
	// Utill class with static methods
	/* Expects:
			/includes
				chatlog_include.php functions
				knowledge_base_include.php
			/vendor
				Yethee\Tiktoken\EncoderProvider() from autoload

	*/

    public static function getAllChatbotShortcodeAttributes(){
		// All Chatbot Shortcode attributes
        return [
			'type' => ['label' => 'Chatbot Type','type'=>'select','options'=>['avatar','non_avatar'],'default_val'=>'avatar','description'=>'3D Talking Avatar or Non-Avatar.'],
			'avatar_character' => ['label'=>'Avatar Character','type'=>'select', 'options'=>['','aws:Cristine','aws:Wes'], 'default_val'=>'', 'description'=>'"provider:character_name", eg: "aws:Cristine", "aws:Wes". NOTE: You may need to download and install the Avatar if it doesnt exist on your server.'],
			'style' => ['label'=>'Style','type'=>'select','options'=>['embedded','floating'], 'default_val'=>'embedded','description'=>'Embedded Chatbots will mostly fill the post/page, whereas Floating will be positioned in the lower right corner.'],
			'floating_title' => ['label'=>'Floating Title','type'=>'text', 'description'=>'Title text visible on Floating chatbot in minimized states. This would typically be short, eg: "Talk to me" or "How can I help?"'],
			'starting_comments' => ['label'=>'Starting Comments/Greeting','type'=>'text', 'description'=>'eg: "Starting Chatbot comment which is displayed and also spoken by Avatar Chatbots, eg: "Ask me anything about XYZ". NOTE: If this is ommitted, the default comments (saved in "Customize" tab) are used.'],
			'common_questions' => ['label'=>'Common Questions','type'=>'textarea', 'description'=>'Comma separated list of Common Questions which will appear as buttons in the chatbot. Eg: How many breweries are there?,What is the most popular beer?'],
			'extra_instructions' => ['label'=>'Extra Instructions to AI Engine','type'=>'textarea', 'description'=>'eg: "Your name is Fred and your goal is to help the user to evaluate this Wordpress site. Remind the user they can see relevant sources in the Conversation Panel at any time."'],
			'user_fields' => ['label'=>'User fields to collect','min_plan'=>'plus','type'=>'custom','sanitize_fn'=>'twxchat_sanitizeChatbotUserFields','function'=>'twxchat_getEditChatbotUserFieldsHtml','description'=>'Add data fields to be collected by the chatbot from the user. These will be captured in the Chat Log and also sent to the "Update Webhook URL" if entered.'],
			'llm_model' => ['label'=>'OpenAI Model','type'=>'select', 'options'=> array_keys(TWXChatUtil::getModelCosts(null,true)), 'default_val'=>'gpt-4o-mini', 'description'=>'The LLM Model used by OpenAI to make the chatbot queries. NOTE! Non-"mini" models typically cost more to use.'],
			'allow_feedback' => ['min_plan'=>'pro','label'=>'Allow Feedback','type'=>'select','options'=>['true','false','logged_in_editor'], 'default_val'=>'false', 'description'=>'Allows User feedback to Chatbot responses to help train the Chatbot over time. Select the "logged_in_editor" option to only allow feedback when a user with edit priviledges is currently logged-in to Wordpress.'],
			'stream' => ['label'=>'Use Streaming','type'=>'select','options'=>['true','false'], 'default_val'=>'true', 'description'=>'Knowledge Base chatbots only. Allow Chatbot to stream responses. Generally faster than without.'],
			'kb_or_assistant' => ['label' => 'Knowledge Base or OpenAI Assistant?','type'=>'select','options'=>['knowledge_base','openai_assistant'],'default_val'=>'knowledge_base','description'=>'Select where to get knowledge context from \'Knowlege Base\' or linked \'OpenAI Assistant\'.'],
			'kb_source' => ['label'=>'Knowledge Base Source Filter','type'=>'text', 'description'=>"During a User query, filter Knowledge Base results to this source, eg: 'mysite.com' would only match against sources which have 'mysite.com' in them. You can select multiple sources by separating them with a comma, eg: 'mysite.com,jpg' would also inlcude JPG images in the Knowledge Base."],
			'min_similarity' => ['label'=>'Min. Similarity (50-99)','type'=>'number', 'min'=>50, 'max'=>99, 'description'=>'Knowledge Base chatbots only. Minimum similarity match score in the Knowledge Base for the Users query. A higher value means more accurate responses but with less Knowledge Base content to work with.'],
			'flashcards' => ['label'=>'Show Flashcards','type'=>'select','options'=>['true','false'], 'default_val'=>'true', 'description'=>'Show the relevant Knowledge Base sources that were used to answer a user query as popup Flashcards. Note these do not appear during mobile sessions.'],
			//'show_sources' => ['label'=>'Show Sources','type'=>'select','options'=>['true','false','avatar_flashcards'], 'default_val'=>'true', 'description'=>'Show the relevant Knowledge Base sources that were used to answer a query. Use \'avatar_flashcards\' if you want your Avatar chatbot to automatically show sources in Flashcards.'],
			'assistant_id' => ['label'=>'OpenAI Assistant ID','type'=>'text', 'description'=>"Chatbot will get responses from an OpenAI 'Assistant'. Assistant must be connected to the OpenAI API Key (see 'General' tab).",'parent_attr'=>'kb_or_assistant','parent_attr_value'=>'openai_assistant'],
			'voice' => ['label'=>'Voice','type'=>'text', 'description'=>'The Chatbot\'s voice. Format is "provider:voice", eg: "openai:onyx". Use a customized <a href="https://platform.openai.com/docs/guides/text-to-speech/quickstart" target="_blank">OpenAI voice</a>. Google Neural voices (eg: "google:Google italiano" etc) can also be used but the user\'s browser needs to support it otherwise the user\'s default browser voice is used.'],
			'avatar_voice' => ['label'=>'Avatar Voice','type'=>'text', 'description'=>'format: "provider:voice_name:speed" (speed is optional), eg: "aws:Ruth", "aws:Brian:110%".<br/>AWS Voices: (aka Polly) has Neural Voices available <a href="https://docs.aws.amazon.com/polly/latest/dg/neural-voices.html" target="_blank">here</a>, make sure to ue the exact value from the "Name/ID" column. Also you will need an AWS Cognito Pool Identity Key (see "General" tab).'],
			'avatar_image_url' => ['label'=>'Avatar Splash Image URL','type'=>'text','sanitize_fn'=>'esc_url_raw','description'=>'URL to image used in the avatar splash/load page, eg: "https://mywebsite.com/images/splash.jpg"'],
			'avatar_background_url' => ['label'=>'Avatar Background Image URL','type'=>'text','sanitize_fn'=>'esc_url_raw','description'=>'URL to background image behind the avatar, eg: "https://mywebsite.com/images/bkg.jpg". Ensure image is hosted locally or it may not appear.'],
			'floating_image_url' => ['label'=>'Floating Image URL','type'=>'text','sanitize_fn'=>'esc_url_raw','description'=>'URL to image used in the floating chatbot. For Avatar chatbots this will appear when User minimizes the chatbot fully.'],
        ];
    }

	public static function loadWordpress() {
		// Load Wordpress Context - used by Stream Handlers etc
		$dir = __DIR__;
		do {
			if (file_exists($dir . '/wp-load.php')) {
				require_once($dir . '/wp-load.php');
				return true;
			}
		} while ($dir = dirname($dir));
		
		$err = 'Could not find wp-load.php';
		exit(esc_html($err));
	}

	public static function getPluginInfo(){
		$plugin_file = dirname(__DIR__) . '/twx-chat.php';
		$plugin_data = get_file_data($plugin_file, [
			'Name'        => 'Plugin Name',
			'Version'     => 'Version',
			'Description' => 'Description',
			'Plan'        => 'Plan',
		]);
		$plugin_data['slug'] = plugin_basename($plugin_file);
		return $plugin_data;
	}

	public static function getAvatarAssetsRootPath($base = 'basedir'){
		$path = wp_upload_dir()[$base] . '/twx-chat/avatar_assets/';
		return $path;
	}

	public static function getUploadsDir($subDir = '', $trailingSlash = true){
		$dir = wp_upload_dir()['basedir'] . '/twx-chat';
		if($subDir) $dir .= '/' . $subDir;
		if($trailingSlash) $dir .= '/';
		return $dir;
	}

	public static function getUploadsUrl($subDir = '', $fileName = ''){
		$url = wp_upload_dir()['baseurl'] . '/twx-chat';
		if($subDir) $url .= '/' . $subDir;
		if($fileName) $url .= '/'.$fileName;
		return $url;
	}

	public static function getTwxlabsUrl($trailingSlash = true){
		$url = 'https://twxchat.twxlabs.au';
		if($trailingSlash) $url .= '/';
		return $url;
	}

	public function getDatetime($datetime_string = 'now') {
		// I think we can use current_time() instead ??
		$timezone_string = get_option('twx_chat_client_timezone');
		if (!$timezone_string) {
			$timezone_string = get_option('timezone_string');
		}

		try {
			$dt = new DateTime($datetime_string, new DateTimeZone($timezone_string ?: 'UTC'));
			return $dt;
		} catch (Exception $ex) {
			return new DateTime($datetime_string, new DateTimeZone('UTC'));
		}
	}

	public static function getHttpResponse($url, $args = []){

		$pluginInfo = TWXChatUtil::getPluginInfo();

		$headers = [
			'Plugin-Version' => $pluginInfo['Version'],
			'Plugin-Plan' => $pluginInfo['Plan'],
			'Site-URL' => home_url(),
			'WP-Version' => get_bloginfo('version'),
			'PHP-Version' => PHP_VERSION,
			'User-Agent' => 'Mozilla/5.0 (WordPress Plugin)',
		];

		if(isset($args['headers'])){
			$headers = array_merge($headers, $args['headers']); // Later keys overwrite earlier ones
		}
		$args['headers'] = $headers;

		return wp_remote_get($url, $args);
	}

	public static function fetchChatbotAvatars($clientToken) {

		$url = TWXChatUtil::getTwxlabsUrl()."api/get_avatars.php?token=$clientToken"; 

		$response = TWXChatUtil::getHttpResponse($url);

		if (is_wp_error($response)) {
			TWXChatUtil::adminLog('TWXChatUtil::fetchChatbotAvatars() failed: ' . $response->get_error_message(),'a',true);
			return [];
		}

		$code = wp_remote_retrieve_response_code($response);
		$body = wp_remote_retrieve_body($response);

		if ($code !== 200 || empty($body)) {
			TWXChatUtil::adminLog("TWXChatUtil::fetchChatbotAvatars() HTTP $code. Body empty or invalid.",'a',true);
			return [];
		}

		$parsed = json_decode($body, true);
		if (!is_array($parsed)) {
			TWXChatUtil::adminLog("TWXChatUtil::fetchChatbotAvatars() JSON decode failed. Body: $body",'a',true);
			return [];
		}

		if (($parsed['status'] ?? '') === 'error') {
			return [];
		}

		$avatars = $parsed['data'] ?? [];
		$assets_path = TWXChatUtil::getAvatarAssetsRootPath();

		foreach ($avatars as $key => $avatar) {
			if (!empty($avatar['local_assets_path']) && !empty($avatar['model'])) {
				$avatars[$key]['exists'] = file_exists($assets_path . $avatar['local_assets_path'].$avatar['model']);
			}
		}

		return $avatars;
	}

	public static function isVersionUpgrade($currVer, $currPlan, $newVer, $newPlan) {

		if($currVer === $newVer){
			if($currPlan === 'free'){
				if($newPlan !== 'free'){
					return true;
				}else{
					return false;
				}
			}else if($currPlan === 'plus'){
				if($newPlan === 'pro'){
					return true;
				}else{
					return false;
				}
			}else{
				// Current is Pro
				return false;
			}
		}else{
			return version_compare($currVer, $newVer,'<');
		}
	}

	public static function isCustomerPlus(){
		return get_option('twx_chat_customer_plan','') === 'plus';
	}

	public static function isCustomerPro(){
		return get_option('twx_chat_customer_plan','') === 'pro';
	}

	public static function customerPlanAccess($plan){
		switch ($plan) {
			case 'plus':
				return TWXChatUtil::isCustomerPlus() || TWXChatUtil::isCustomerPro();
			case 'pro':
				return TWXChatUtil::isCustomerPro();
		}
		return true; // Free
	}

	public static function getParamsForLLM(
		$apiKey, 
		$threadId, 
		$extraInstructions, 
		$chatbotGreeting, 
		$currentUserMsg, 
		$isCommonQuestion, 
		$knowledgeBaseSource, 
		$model, 
		$minSimilarity, 
		$showSources, 
		$chatbotId, 
		$userFieldsWithVals, 
		$numUserFieldsToCollect) 
	{

		// Main function to return paramteres for an LLM query using RAG etc

		// Check if we need to add Bot Greeting to Chatlog
		// We get strange results if we add this. ie: chatbot doesnt seem to gather first user field in user response 1, and also bot finishes engaging after user fields collected
		/*
		$chatLogEntries = TWXChatLog::getChatLogEntriesByThreadId($threadId);
		if(empty($chatLogEntries) && !empty($chatbotGreeting)){
			$atts = TWXChatUtil::getChatbotAttributesById(['id' => $chatbotId]);
			if(isset($atts['user_fields']) && !empty($atts['user_fields'])){
				TWXChatLog::insertChatLog($threadId, ['botResponse' => $chatbotGreeting]);
			}
		}
		*/

		// If not a Common Question button click and if user fields exist and still need to collect more than 1 field (Plus Plan, otherwise values will be empty)
		if(!$isCommonQuestion && $numUserFieldsToCollect > 1 && !empty($userFieldsWithVals)){
			// return message history with instructions geared towards collecting missing user fields
			$newMessages = TWXChatUtil::getMessagesForLLM_CollectUserFields($threadId, $chatbotGreeting, $currentUserMsg, $userFieldsWithVals, $numUserFieldsToCollect);
			return [$newMessages, null, null, null];
		}

		$lastNPairs = 3;
		$systemPrompt = TWXChatUtil::getLLMSystemPrompt($extraInstructions, $userFieldsWithVals, $numUserFieldsToCollect);
		
		list($prevMessages, $prevSources) = TWXChatLog::getMessagesAndSourcesFromChatLog($threadId, $systemPrompt, $chatbotGreeting, $currentUserMsg, $lastNPairs); // get max last N message pairs and sources
		//TWXChatUtil::log("TWXChatUtil.getParamsForLLM:: prev messages for thread $threadId: ".var_export($prevMessages,true));

		// get Knowledge Base entries
		$kbEntries = [];
		if(!empty($knowledgeBaseSource)){
			$kbEntries = TWXChatKnowledgeBaseUtil::getEntriesBySource($knowledgeBaseSource, false);
		}else{
			$kbEntries = TWXChatKnowledgeBaseUtil::getEntriesNoSource();
		}
	
		TWXChatUtil::log('TWXChatUtil::getMessagesForLLM() Total knowledge base entries'.(empty($knowledgeBaseSource) ? '' : ' for "'.$knowledgeBaseSource.'"').': '.count($kbEntries));
	
		$tiktokenProvider = new Yethee\Tiktoken\EncoderProvider(); // from vendor autoload
	
		// determine how many input tokens we have available
		$modelCosts = TWXChatUtil::getModelCosts($model);
	
		$maxTokens = -1;
		if(isset($modelCosts['max_tokens'])){
			$prevMessagesTokens = count(TWXChatUtil::getTokens($tiktokenProvider, $model, json_encode($prevMessages)));
			$maxTokens = intval(($modelCosts['max_tokens'] - $prevMessagesTokens) * 0.95); // only use 95% of maximum available tokens, can reduce this to be more cost effective but may decrease quality of response
		}
	
		$userMessages = TWXChatUtil::getUserPromptsReverse($prevMessages);
	
		$prevSourceEntries = [];
		if(!empty($prevSources)){
			foreach($prevSources as $id => $prevSource){
				if(isset($kbEntries[$id])) $prevSourceEntries[$id] = $kbEntries[$id];
			}
		}
	
		// get Knowledge Base Entries to use for RAG. If is a follow up q and previous sources exist, use those instead of all entries search
		$similarKnowledgeBaseEntries = [];
		if(TWXChatUtil::isFollowUpQuery($currentUserMsg) && count($prevSourceEntries) > 0 ){
			TWXChatUtil::log("TWXChatUtil::getParamsForLLM() is follow-up query, use previous Knowledge Base entries for RAG");
			$similarKnowledgeBaseEntries = $prevSourceEntries;
		}else{
			$similarKnowledgeBaseEntries = TWXChatKnowledgeBaseUtil::searchSimilarKnowledgeBaseEntries($tiktokenProvider, $apiKey, $userMessages, $kbEntries, [], $model, $maxTokens, $minSimilarity);
			// If no matching Knowledge Base entries but we have previous Knowledge Base sources, lets use those just in case isFollowUpQuery() didn't work
			if(empty($similarKnowledgeBaseEntries) && count($prevSourceEntries) > 0){
				$similarKnowledgeBaseEntries = $prevSourceEntries;
			}
		}
		TWXChatUtil::log("TWXChatUtil::getParamsForLLM() similarKnowledgeBaseEntries = ".count($similarKnowledgeBaseEntries));
	
		$context = TWXChatKnowledgeBaseUtil::getContextFromEntries($similarKnowledgeBaseEntries, true);
		TWXChatUtil::log("TWXChatUtil.getParamsForLLM:: RAG context length = ".strlen($context));
	
		$newUserPrompt = TWXChatUtil::getLLMUserPrompt($currentUserMsg, $context, $showSources, $userFieldsWithVals);

		// get previous messages and add new modified one
		$newMessages = $prevMessages;
		if(!empty($newMessages)) array_pop($newMessages);  // remove last message as it is the current one and we need to enhance it
		$newMessages[] = ['role' => 'user', 'content' => $newUserPrompt];

		$contentAll = array_column($newMessages,'content');
		$tokens = TWXChatUtil::getTokens($tiktokenProvider, $model, implode(" ",$contentAll));
		$estimated['tokens'] = count($tokens);
		
		//TODO check when/if these are used as we are already passing $similarKnowledgeBaseEntries back
		$sources = [];
		foreach($similarKnowledgeBaseEntries as $id => $entry){
			$sources[$id] = [
				'source_id' => $id, 
				'source' => $entry['source'], 
				'similarity_score' => $entry['similarity_score'], 
				'content_length_chars' => strlen($entry['orig_text']),
				'last_modified' => $entry['created_at'],
			];
		}

		return [$newMessages, $similarKnowledgeBaseEntries, $estimated, $sources];
	}

	public static function getMessagesForLLM_CollectUserFields($threadId, $chatbotGreeting, $currentUserMsg, $userFieldsWithVals, $numUserFieldsToCollect){

		// Return Messages to be used in LLM query in order to collect missing user fields
		// Assumes $userFieldsWithVals not empty and still more user fields to collect

		$lastNPairs = null; //count($userFieldsWithVals);

		$systemPrompt = TWXChatUtil::getLLMSystemPrompt(null, $userFieldsWithVals, $numUserFieldsToCollect);

		// Below is a chatlog_include function
		list($prevMessages, $prevSources) = TWXChatLog::getMessagesAndSourcesFromChatLog($threadId, $systemPrompt, $chatbotGreeting, $currentUserMsg, $lastNPairs);

		$newUserPrompt = TWXChatUtil::getLLMUserPrompt($currentUserMsg, null, null, $userFieldsWithVals);

		$newMessages = $prevMessages;
		if(!empty($newMessages)) array_pop($newMessages);  // remove last message as it is the current one and we need to enhance it
		$newMessages[] = ['role' => 'user', 'content' => $newUserPrompt];

		return $newMessages;
	}

	public static function getLLMSystemPrompt($extraInstructions, $userFieldsWithVals, $numUserFieldsToCollect = 0){

		// Construct a System Prompt based on extra instructions and user fields (if they exist)
		/*
		$userFieldsString = '';
		if(!empty($userFieldsWithVals)){
			for($i=0; $i<count($userFieldsWithVals); $i++){
				$field = $userFieldsWithVals[$i];
				$userFieldsString .= ($i+1) . '. ' . $field['name'];
				if(isset($field['description']) && !empty($field['description'])) {
					$userFieldsString .= ' ('.$field['description'] .')';
				}
				$userFieldsString .= "\n";
			}
		}
		*/	

		$prompt = '';

		if(!empty($extraInstructions)) {
			$prompt .= $extraInstructions . "\n\n";
		}else{
			$prompt .= "You are a helpful assistant.\n\n";
		}

		if(!empty($userFieldsWithVals)){

			$jsonUserFields = json_encode($userFieldsWithVals, JSON_PRETTY_PRINT);

			$prompt .= "Another objective is to collect the following fields from the user and use that information as context for the above instructions. Politely keep asking for missing fields.\n\n"
				. "User Fields:\n"
				. $jsonUserFields . "\n\n"
				. "As user field values are detected or changed in user messages, you need to capture these, and possibly other data, in a JSON object at the end of each response starting with a § character, for example:\n"
				. "§{\"user_fields\":{\"user_name\":\"John\",\"user_email\":\"john@example.com\"},\"source_ids\":[\"id1\",\"id34\",\"etc\"],\"other_data\":\"etc\"}\n\n"
				. "User prompts may contain additional instructions.\n";
		}

		return $prompt;

		//TODO remove below as needed
		// If user fields exist and still more to be collected this takes precedence. Will be used in Chat Completions Query only (not streaming)

		if($numUserFieldsToCollect > 0){
			return "Your objective is to collect the following information from the user, preferably in the following order:  " . $userFieldsString . "\n"
				. "You MUST respond to every user message by calling the `user_fields_main_function`. Do not speak directly to the user. \n"
				. "The `user_fields_main_function` requires a parameter called `full_response`. This must contain the full user-facing reply you would normally say.\n"
				. "If you detect user fields, also include those fields in the function call.\n"
				. "Never reply directly. Only use `user_fields_main_function`."
				;
		}

		/*
		if(!empty($userFieldsWithVals) && $numUserFieldsToCollect > 1){
			return 'Your objective is to collect the following information from the user, preferably in the following order: ' . $userFieldsString . '. '
				.'You will pleasantly continue to ask for any missing fields.';
		}
		*/

		if(!empty($extraInstructions) && !empty($userFieldsWithVals)){
			return "Your first objective is to collect the following information from the user, preferably in the following order:\n" . $userFieldsString . "\n"
				. "Do not confirm the collected fields with the user. The user may change field values at any time.\n"
				//. "At the end of your response (when any user_fields have been detected or updated), always include a JSON object on a new line starting with the § character.\n"
				//. "Example: §{". TWXChatUtil::getUserFieldsLLMAnswerTemplate($userFieldsWithVals)."}\n"
				//. "Only include keys for user fields you have detected. Do not fabricate or use example placeholder values.\n\n"
				. "After collecting all user fields, please continue with these instructions:\n$extraInstructions"
				;
		}

		if(empty($extraInstructions) && !empty($userFieldsWithVals)){
			return "Your objective is to collect the following information from the user, preferably in the following order:\n" . $userFieldsString . "\n"
				. "Do not confirm the collected fields with the user. The user may change field values at any time.\n"
				//. "At the end of your response (when any user_fields have been detected or updated), always include a JSON object on a new line starting with the § character.\n"
				//. "Example: §{". TWXChatUtil::getUserFieldsLLMAnswerTemplate($userFieldsWithVals)."}\n";
				//. "Only include keys for user fields you have detected. Do not fabricate or use example placeholder values like '{new value}'.";
				;
		}

		if(!empty($extraInstructions)){
			return $extraInstructions;
		}

		return 'You are a helpful assistant';
	}

	public static function getLLMUserPrompt($currentUserMsg, $context, $showSources, $userFieldsWithVals){

		// Return modified user prompt. User fields may exist and if they do we can assume they have been fully collected, but could be changed

		$prompt = '';

		if(!empty($context) && empty($userFieldsWithVals)){
			$prompt = "Based on the sources below, answer the following question:\n$currentUserMsg\n\n" 
                ."Do not make statements like 'according to the sources' or 'based on the sources' etc.\n\n";
			//TODO
			if($showSources && $showSources === 'avatar_flashcards'){
				/*
				$prompt .= "You are an avatar chatbot with SSML capabilities. Feel free to insert SSML within your response. For example if you want the avatar to look at the Flashcards (which contain sources) you could insert something like:\n"
				. ' For more information click on the sources <mark name=\'{"feature":"PointOfInterestFeature","method":"setTargetByName","args":["Flashcards"]}\'/> here.' . "\n\n";
				*/
			}
			$prompt .= "At the end of your answer, return the source_ids used to generate the response as a JSON string on a new line, starting with a § character exactly like this:\n"
				.'§{"source_ids":["id1","id4","id25","etc"]}'
				."\nThe source_ids should be in relevance order from highest to lowest.\n\n";

			$prompt .= "Sources:\n$context";
		}

		$userFieldsAnswerTemplate = TWXChatUtil::getUserFieldsLLMAnswerTemplate($userFieldsWithVals);

		if(!empty($context) && !empty($userFieldsWithVals)){
			//$prompt = 'Current User Fields: '.json_encode($userFieldsWithVals)."\n\n";
			$prompt = "Based on the sources below and the current user fields, answer the following question:\n$currentUserMsg\n\n" 
                ."Do not make statements like 'according to the sources' or 'based on the sources' etc.\n\n";
			//TODO
			if($showSources && $showSources === 'avatar_flashcards'){
				/*
				$prompt .= "You are an avatar chatbot with SSML capabilities. Feel free to insert SSML within your response. For example if you want the avatar to look at the Flashcards (which contain sources) you could insert something like:\n"
				. ' For more information click on the sources <mark name=\'{"feature":"PointOfInterestFeature","method":"setTargetByName","args":["Flashcards"]}\'/> here.' . "\n\n";
				*/
			}
			$prompt .= "Remember, at the end of your answer, return the source_ids used to generate the response, and any changes to user fields, as a JSON object on a new line, starting with a § character exactly like this:\n"
				.'§{"source_ids":["id1","id4","id25","etc"],'.$userFieldsAnswerTemplate.'}'
				."\nThe source_ids should be in relevance order from highest to lowest.\n\n";

			$prompt .= "Sources:\n$context";
		}

		if(empty($context) && !empty($userFieldsWithVals)){
			//$prompt = 'Current User Fields: '.json_encode($userFieldsWithVals)."\n\n";
			$prompt = "Based on the current user fields, answer the following question:\n$currentUserMsg\n\n" 
                ."Do not make statements like 'according to the fields' or 'based on the fields' etc.\n"
				. "Remember to include user field values at the end in a JSON object.";
			/*
			$prompt .= "Remember, at the end of your answer, return any changes to user fields as a JSON string on a new line, starting with a § character exactly like this:\n"
				.'§{'.$userFieldsAnswerTemplate.'}';
			*/
		}

		return $prompt;
	}

	public static function getUserFieldsLLMAnswerTemplate($userFieldsWithVals){

		if(empty($userFieldsWithVals)) return '';
		$answerTemplate = [];
		foreach($userFieldsWithVals as $field){
			$fieldName = $field['name'] ?? '';
			if(empty($fieldName)) continue;
			$fieldValue = $field['value'] ?? '';
			$answerTemplate['user_fields'][$fieldName] = (!empty($fieldValue) ? $fieldValue : 'value');
		}

		return substr(json_encode($answerTemplate), 1, -1);
	}

	public static function getUserPromptsReverse($messages){

		if(empty($messages)) return [];
		//if(count($messages)==1) return [];
		
		$ret = [];
	
		for($i=count($messages)-1; $i>=0; $i--){
			$msg = $messages[$i];
			if($msg['role'] == 'user'){
				$ret[] = TWXChatUtil::appendQuestionMark($msg['content']);
			}
		}
	
		return $ret;
	}

	public static function getLastBotMessageContent($messages) {
		if(empty($messages)) return null;
		for ($i = count($messages) - 1; $i >= 0; $i--) {
			if ($messages[$i]['role'] === 'assistant') {
				return $messages[$i]['content'];
			}
		}
		return null; // none found
	}
	
	public static function splitDataFromBotResponse($botResponse){
		if(empty($botResponse)) return ['',''];
		//$botResponse = mb_convert_encoding($botResponse, 'UTF-8', 'UTF-8');
		$lastDelimPos = strrpos($botResponse, '§');

		if ($lastDelimPos !== false) {
			$before = trim(substr($botResponse, 0, $lastDelimPos));
			$after = trim(substr($botResponse, $lastDelimPos + 2)); // for some reason +1 is leading with a strange char �, +2 works ok
		} else {
			$before = $botResponse;
			$after = '';
		}
		return [$before, $after];
	}

	public static function getModelCosts($model = null, $llmOnly = false) {
		
		$models = [
			'gpt-4o-mini' => ['in'=>0.0000005,'out'=>0.0000006, 'max_tokens' => 128000],
			'gpt-4o-mini-2024-07-18' => ['in'=>0.0000005,'out'=>0.0000006, 'max_tokens' => 128000],
			'gpt-3.5-turbo' => ['in'=>0.0000005,'out'=>0.0000015, 'max_tokens' => 16385],
			'gpt-3.5-turbo-0125' => ['in'=>0.0000005,'out'=>0.0000015, 'max_tokens' => 16385],
			'gpt-3.5-turbo-1106' => ['in'=>0.0000005,'out'=>0.0000015, 'max_tokens' => 16385],
			'gpt-4o' => ['in'=>0.000005,'out'=>0.000015, 'max_tokens' => 128000],
			'gpt-4-turbo' => ['in'=>0.000005,'out'=>0.000015, 'max_tokens' => 128000],
			//'embedding' => ['in'=>0.00000002,'out'=>0.00000013]
		];

		if(!$llmOnly) $models['embedding'] = ['in'=>0.00000002,'out'=>0.00000013];
		
		if(empty($model)) return $models;
		
		if(isset($models[$model])) return $models[$model];
		
		return null;
	}
	
	public static function getEmbedding($apiKey, $text)
	{

		$url = 'https://api.openai.com/v1/embeddings';
		
		$data = [
			'input' => $text,
			'model' => 'text-embedding-ada-002'
		];
		
		$response = self::getHttpOpenAI_WP($apiKey, $url, $data, 'POST');

		if(isset($response['error'])){
			throw new Exception(esc_html($response['error']['message']));
		}

		if(isset($response['data'][0]['embedding'])) return $response['data'][0]['embedding'];
		
		return [];
	}	
	
	public static function cosineSimilarity($embedding1, $embedding2)
	{
		if(empty($embedding1) || empty($embedding2)) return 0;
		$dotProduct = 0.0;
		$normA = 0.0;
		$normB = 0.0;
		for ($i = 0; $i < count($embedding1); $i++) {
			$dotProduct += $embedding1[$i] * $embedding2[$i];
			$normA += pow($embedding1[$i], 2);
			$normB += pow($embedding2[$i], 2);
		}
		return $dotProduct / (sqrt($normA) * sqrt($normB));
	}	
	
	public static function getTokens(&$tiktokenProvider, $model, $text) {

		// requires Yethee\Tiktoken library
		try{
			$encoder = $tiktokenProvider->getForModel($model);
			$tokens = $encoder->encode($text);
			return $tokens;
		}catch(Exception $ex){
		}
		return [];
	}
	
	public static function isFollowUpQuery($text) {
		$text = strtolower(trim($text));
	
		// List of vague pronouns or references
		$contextualWords = ['they', 'them', 'it', 'those', 'these', 'that', 'he', 'she', 'their'];
	
		// Check for presence of any of the above words anywhere in the sentence
		foreach ($contextualWords as $word) {
			if (preg_match('/\b' . preg_quote($word, '/') . '\b/', $text)) {
				return true;
			}
		}
	
		// Optional: Short question detection (often contextual)
		/*
		if (str_word_count($text) < 4) {
			return true;
		}
		*/
	
		return false;
	}

	public static function isValidJson($string) {
		json_decode($string);
		return json_last_error() === JSON_ERROR_NONE;
	}

	public static function sanitizeJS($input) {
		// Remove any <script> tags (opening or closing)
		return preg_replace('#<\s*/?\s*script[^>]*>#i', '', $input);
	}	

	public static function sanitizeCSS($input) {
    	// Remove <style> tags if present (they're unnecessary in inline CSS)
    	return preg_replace('#<\s*/?\s*style[^>]*>#i', '', $input);		
	}	

	/**
	 * Sanitize and validate uploaded file array.
	 *
	 * @param array $file The $_FILES['your_input'] array.
	 * @param int   $max_size Maximum allowed file size in bytes (default 5MB).
	 * @param array $allowed_types Optional array of allowed MIME types/extensions.
	 *
	 * @return array|WP_Error Sanitized file array or WP_Error on failure.
	 */
	public static function sanitizeFileUpload( $file, $max_size = 5242880, $allowed_types = [] ) {
		if ( empty( $file ) || ! isset( $file['name'] ) || ! isset( $file['tmp_name'] ) ) {
			return new WP_Error( 'no_file', __( 'No file uploaded.', 'twx-chat' ) );
		}

		// Sanitize filename
		$file['name'] = sanitize_file_name( wp_unslash( $file['name'] ) );

		// Validate file size
		if ( $file['size'] > $max_size ) {
			return new WP_Error( 'file_too_large', __( 'Uploaded file exceeds size limit.', 'twx-chat' ) );
		}

		// Validate file type and extension
		$filetype = wp_check_filetype_and_ext( $file['tmp_name'], $file['name'] );

		if ( ! $filetype['ext'] || ! $filetype['type'] ) {
			return new WP_Error( 'invalid_file_type', __( 'Invalid file type.', 'twx-chat' ) );
		}

		// If allowed_types specified, enforce
		if ( ! empty( $allowed_types ) && ! in_array( $filetype['type'], $allowed_types, true ) ) {
			return new WP_Error( 'disallowed_file_type', __( 'File type not allowed.', 'twx-chat' ) );
		}

		// Replace detected type
		$file['type'] = $filetype['type'];

		return $file;
	}

	public static function obscureApiKey($key, $prefix = ''){
		if(empty($key)) return '';
		$key = trim($key);
		if(empty($key)) return '';
		if(strlen($key)<6) return '******';
		return $prefix.'**********'.substr($key, -6);
	}

	public static function urlExists($url) {
		if (!filter_var($url, FILTER_VALIDATE_URL)) {
			return false;
		}

		$response = TWXChatUtil::getHttpResponse($url, [
			'method'      => 'GET',
			'timeout'     => 10,
			'redirection' => 5,
			'limit_response_size' => 2048, // Optional: limit response size to save memory
		]);

		if (is_wp_error($response)) {
			TWXChatUtil::log("TWXUtil::urlExists(): error for $url - " . $response->get_error_message(), 'a', true);
			return false;
		}

		$status_code = wp_remote_retrieve_response_code($response);
		return ($status_code >= 200 && $status_code < 400);
	}

	public static function synthesizeSpeech($apiKey, $text, $outputFile, $voice = 'onyx') {
		
		// Endpoint URL
		$url = 'https://api.openai.com/v1/audio/speech';

		$data = [
			'model' => 'tts-1',
			'input' => $text,
			'voice' => $voice,
		];
		
		$response = self::getHttpOpenAI_WP($apiKey, $url, $data, 'POST', false);

		if(self::isValidJson($response)){
			
			$response = json_decode($response,true);
			if(isset($response['error'])){
				return $response['error'];
			}else{
				return $response;
			}
			
		}else{
			
			file_put_contents($outputFile, $response);
			return file_exists($outputFile);
		}

	}
	
	/*
	public static function goCURL($apiKey, $url, $data, $method = 'POST', $decodeResponse = true) {
		
		// Perform HTTP GET or HTTP POST (using json data) - Using CURL Here as this Method may be used in other Non-WP Projects
		
		$curl = curl_init();
		
		$header = array(
			'Authorization: Bearer '.$apiKey,
			'Content-Type: application/json',
			'OpenAI-Beta: assistants=v2'
		);

		$opts = [
		  CURLOPT_URL => $url,
		  CURLOPT_RETURNTRANSFER => true,
		  CURLOPT_ENCODING => '',
		  CURLOPT_MAXREDIRS => 10,
		  CURLOPT_TIMEOUT => 0,
		  CURLOPT_FOLLOWLOCATION => true,
		  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
		  CURLOPT_CUSTOMREQUEST => $method,
		  CURLOPT_HTTPHEADER => $header,
		];
		
		if($method == 'POST') {
			$opts[CURLOPT_POST] = true;
			$opts[CURLOPT_POSTFIELDS] = json_encode($data);
		}else if($method == 'GET' && !empty($data)){
			// Append data as params to URL
			$newURL = $url;
			if (strpos($url, '?') === false) {
				// No query string, just append the encoded parameters
				$newURL = $url . '?' . http_build_query($data);
			} else {
				// There is an existing query string, append with an ampersand
				$newURL = $url . '&' . http_build_query($data);
			}
			$opts[CURLOPT_URL] = $newURL;
		}

		curl_setopt_array($curl, $opts);

		$response = curl_exec($curl);

		curl_close($curl);
		
		if($decodeResponse) return json_decode($response, true);
		return $response;
	}
	*/

	public static function getHttpOpenAI_WP($apiKey, $url, $data = [], $method = 'POST', $decodeResponse = true) {

		// OpenAI HTTP Request Similar to goCURL but can be used within Wordpress environments
		
		$headers = [
			'Authorization' => 'Bearer ' . $apiKey,
			'Content-Type' => 'application/json',
			'OpenAI-Beta' => 'assistants=v2',
		];

		$args = [
			'headers' => $headers,
			'timeout' => 30,
		];

		if ($method === 'POST' || $method === 'PATCH') {
			$args['body'] = json_encode($data);
			$response = wp_remote_post($url, $args);
		} else if ($method === 'GET') {
			if (!empty($data)) {
				$url = add_query_arg($data, $url);
			}
			$response = wp_remote_get($url, $args);
		} else {
			return false; // unsupported method
		}

		if (is_wp_error($response)) {
			return false;
		}

		$body = wp_remote_retrieve_body($response);

		return $decodeResponse ? json_decode($body, true) : $body;
	}
	
    public static function parseAnswer($in){
		
		$out = $in;
		
		$out = preg_replace('/【.*?】/', '', $out); // remove annoations/sources for the moment
		
		$out = str_ireplace('***','',$out);
		
		$out = str_ireplace('**','',$out);
		
		$out = str_ireplace('###','',$out);
		
		$out = str_ireplace('##','',$out);

		return $out;
	}
	
	public static function validateAnswerWithKnowledgeBaseEntries($answer, &$entries){
		
		// attempt to reduce AI Hallucinations by analysing AI Answer (Response) with the Knowledge Base entries used in the input prompt
		$ret = $answer;
		
		// First Check Hyperlinks
		$ret = self::updateHyperlinks($ret, $entries);
		
		return $ret;
	}
	
    private static function updateHyperlinks($html, &$entries, $removeLinkIfNotFound = true) {
        // Define a regex pattern to match <a> tags and capture href
        $pattern = '/<a\s+([^>]*?)href=["\']([^"\']+)["\']([^>]*?)>(.*?)<\/a>/i';
		
        // Callback function to process matches
        $callback = function($matches) use (&$entries, $removeLinkIfNotFound) {
			
            $beforeHref = $matches[1];
            $href = $matches[2];
            $afterHref = $matches[3];
            $text = $matches[4];

			$newHref = '';
            
			foreach($entries as $entry){
				$markdown = $entry['orig_text'];
				$source = $entry['source'];
				$pos = stripos($markdown,$text);
				
				//TWXChatUtil::log('checking in source: '.$source);

				if($pos !== false){
					//TWXChatUtil::log('found mention of: '.$text);
					
					$newHref = self::findMarkdownLinksWithText($markdown, $text);
					if(!empty($newHref)){
						break;
					}
				}
			}
			
			if(!empty($newHref)){
				return '<a ' . $beforeHref . 'href="' . htmlspecialchars($newHref) . '"' . $afterHref . '>' . htmlspecialchars($text) . '</a>';
			}

			if($removeLinkIfNotFound) return $text;

			return '<a ' . $beforeHref . 'href="' . htmlspecialchars($href) . '"' . $afterHref . '>' . htmlspecialchars($text) . '</a>';
        };

        // Perform the regex replacement
        $updatedHtml = preg_replace_callback($pattern, $callback, $html);

        return $updatedHtml;
    }

	public static function findMarkdownLinksWithText($markdown, $searchText) {
		// Define a regex pattern to match markdown links and capture the text and URL
		$pattern = '/\[(.*?)\]\((.*?)\)/i';
		
		// Initialize an array to hold matching markdown links
		$matches = [];
		
		// Perform the regex match
		if (preg_match_all($pattern, $markdown, $found)) {
			foreach ($found[1] as $index => $text) {
				// Check if the search text matches the hyperlink text
				if (stripos(trim($text), $searchText) !== false) {
					// Store the entire markdown link portion
					return $found[2][$index];
					//$matches[] = $found[0][$index];
				}
			}
		}
		
		return '';
	}
	
	public static function extractMarkdownImages($markdown) {
		$images = [];

		// Match ![alt text](image-url)
		preg_match_all('/!\[(.*?)\]\((.*?)\)/', $markdown, $matches, PREG_SET_ORDER);

		foreach ($matches as $match) {
			$images[] = [
				'alt' => $match[1],
				'url' => $match[2],
			];
		}

		return $images;
	}

	public static function extractBlurbFromMarkdown($markdown, $maxLength = 150) {
		// Remove image tags: ![alt](url)
		$markdown = preg_replace('/!\[.*?\]\(.*?\)/', '', $markdown);

		// Remove links but keep link text: [text](url) → text
		$markdown = preg_replace('/\[(.*?)\]\(.*?\)/', '$1', $markdown);

		// Strip other Markdown syntax (*, _, `, >, etc.)
		$markdown = wp_strip_all_tags($markdown); // basic HTML tags (if any)
		$markdown = preg_replace('/[_*`>#-]/', '', $markdown); // common MD chars
		$markdown = trim($markdown);

		// Split into lines/paragraphs
		$lines = preg_split('/\r\n|\r|\n/', $markdown);
		
		foreach ($lines as $line) {
			$line = trim($line);
			if (!empty($line)) {
				// Truncate to maxLength if needed
				if (strlen($line) > $maxLength) {
					$line = substr($line, 0, $maxLength);
					$line = preg_replace('/\s+\S*$/', '', $line); // Avoid cutting mid-word
					$line .= '...';
				}
				return $line;
			}
		}

		return '';
	}

	public static function parseInlineSourcesOLD($text){

		$uniqueIds = [];

		// Match any word or phrase immediately followed by §[...]
		$pattern = '/(\b.+?)§\[(.*?)\]/';
	
		$processedText = preg_replace_callback($pattern, function ($matches) use (&$uniqueIds) {
			$label = trim($matches[1]);      // Text before the §
			$rawIds = $matches[2];           // Contents inside brackets
	
			// Attempt to parse the array
			$ids = json_decode("[$rawIds]", true);
			if (!is_array($ids)) return $matches[0]; // Leave unchanged if parsing fails
	
			// Collect unique source IDs
			foreach ($ids as $id) {
				if (!in_array($id, $uniqueIds)) {
					$uniqueIds[] = $id;
				}
			}
	
			$dataAttr = htmlspecialchars(implode(',', $ids), ENT_QUOTES);
			return '<span class="inline-source" data-source-ids="' . $dataAttr . '">' . $label . '</span>';
		}, $text);
	
		return [
			'parsed' => $processedText,
			'source_ids' => $uniqueIds
		];		
	}

	public static function stripInLineSources($text){
		return rtrim(strstr($text, '§', true));
	}

	public static function parseInlineSources($text) {

		/*
		in: Here is an inline source§["1766"], followed by two inline sources§["1234","1766"]. I hope that helps!
		out: Here is an inline <span class="inline-source" data-source-ids="1766">source</span>, followed by two inline <span class="inline-source" data-source-ids="1234,1766">sources</span>. I hope that helps!

		Also returns an array of unique source ids: eg: ["1766","1234"];
		*/

		$uniqueIds = [];

		// Match a word (including punctuation) immediately before §[...] — avoids greediness
		$pattern = '/(\S+?)§\[(.*?)\]/';
	
		$processedText = preg_replace_callback($pattern, function ($matches) use (&$uniqueIds) {
			//TWXChatUtil::log('parseInlineSources. matches: '.var_export($matches,true));
			$label = $matches[1];
			$rawIds = $matches[2];
	
			// Decode source IDs (comma-separated)
			$ids = json_decode("[$rawIds]", true);
			//TWXChatUtil::log('parseInlineSources. ids: '.var_export($ids,true));
			if (!is_array($ids)) return $matches[0]; // Return original if decoding fails
	
			// Add to unique IDs
			foreach ($ids as $id) {
				if (!in_array($id, $uniqueIds)) {
					$uniqueIds[] = $id;
				}
			}
			$dataAttr = htmlspecialchars(implode(',', $ids), ENT_QUOTES);
			$span = '<span class="inline-source" data-source-ids="' . $dataAttr . '">' . $label . '</span>'; 

			return $span; //'<span class="inline-source" data-source-ids="' . $dataAttr . '">' . $label . '</span>';
		}, $text);
	
		return [$processedText, $uniqueIds];
	}	

	public static function extractSourceIdsFromText($text) {
		$sourceIds = [];
	
		// Match all spans with class "inline-source" and extract data-source-id
		$pattern = '/<span[^>]*class="[^"]*inline-source[^"]*"[^>]*data-source-id="([^"]+)"[^>]*>/i';
	
		if (preg_match_all($pattern, $text, $matches)) {
			foreach ($matches[1] as $idString) {
				// If multiple IDs are comma-separated, split them
				$ids = array_map('trim', explode(',', $idString));
				$sourceIds = array_merge($sourceIds, $ids);
			}
		}
	
		// Return unique IDs
		return array_values(array_unique($sourceIds));
	}
	

	public static function parseLinks($text) {
		
		/* eg:
			Seeker Brewing - [Seeker Brewing Website](https://mysite.net.au/seeker) and ABC Ales [ABC Brewing](https://mysite.net.au/abc) etc could be (lots) of text with [brackets] etc

			to

			Seeker Brewing - <a href="https://mysite.net.au/seeker" target="_new">Seeker Brewing Website</a> and ABC Ales <a href="https://mysite.net.au/abc" target="_new">ABC Brewing</a> etc

			parts:

				Seeker Brewing - [Seeker Brewing Website
				s://mysite.net.au/seeker) and ABC Ales [ABC Brewing
				s://mysite.net.au/abc) etc could be (lots) of text with [brackets] etc
		*/
		
		$ret = '';
		
		$delim = '](http';
		$split = explode($delim, $text);
		$linkText = '';
		$linkUrl = '';
		$leftLinkSurround = '';  // [
		$rightLinkSurround = '';  // ]
		
		for($i=0;$i<count($split);$i++){
			
			$part = $split[$i];
			
			if($i==0){
				// start part
				$linkText = '';
				$sb_pos = strrpos($part,'[');
				if($sb_pos !== false){
					$ret.= substr($part,0,$sb_pos);
					$ret.= $leftLinkSurround.'<a target="_new" href="';
					$linkText = substr($part,$sb_pos+1);
				}else{
					$ret.= $part;
				}
			} elseif($i == count($split)-1) {
				// end part
				if(!empty($linkText)){
					$rb_pos = strpos($part,')');
					if($rb_pos !== false){
						$linkUrl = 'http'.substr($part,0,$rb_pos);
						$ret.= $linkUrl.'">'.$linkText.'</a>'.$rightLinkSurround;
						$ret.= substr($part,$rb_pos+1);
					}else{
						$ret.= $part;
					}
				}else{
					$ret.= $part;
				}
			} else {
				// middle part(s)
				if(!empty($linkText)){
					$rb_pos = strpos($part,')');
					if($rb_pos !== false){
						$linkUrl = 'http'.substr($part,0,$rb_pos);
						$ret.= $linkUrl.'">'.$linkText.'</a>'.$rightLinkSurround;
						$sb_pos = strrpos($part,'[');
						if($sb_pos !== false){
							$ret.= substr($part,$rb_pos+1,($sb_pos-$rb_pos-1)); 
							$ret.= $leftLinkSurround.'<a target="_new" href="';
							$linkText = substr($part,$sb_pos+1);
						}else{
							$ret.= substr($part,$rb_pos+1);
							$linkText = '';
						}
					}else{
						$ret.= $part;
					}
				}else{
					$ret.= $part;
				}
					
			}
			
		}
		
		return $ret;
		
	}

	public static function getChatbotAttributesFromPost($post) {
		if (!isset($post->post_content)) {
			return [];
		}

		$shortcode_tag = 'twxchat';
		$pattern = get_shortcode_regex([$shortcode_tag]);
		$results = [];

		if (preg_match_all('/' . $pattern . '/s', $post->post_content, $matches, PREG_SET_ORDER)) {
			foreach ($matches as $shortcode) {
				$atts = shortcode_parse_atts($shortcode[3]);
				if (is_array($atts)) {
					// Merge with existing attributes (later ones override)
					$results = array_merge($results, $atts);
				}
			}
		}
				
		return $results;
	}


	public static function getChatbotAttributesById($atts, $mergeWithAll = false) {

		/*
		$atts needs to contain 'id' but may also contain override atts, eg: twx_debug etc 
		*/

		if(!isset($atts['id'])) return [];
		$id = $atts['id'];

		global $wpdb;
		$table_name = $wpdb->prefix . 'twxchat_chatbots';

		$row = null;
		if ($id) {
			$row = $wpdb->get_row($wpdb->prepare("SELECT * FROM $table_name WHERE id = %d", $id));
			if(!$row) return [];
			$chatbot_atts = json_decode($row->settings, true);
			$atts = array_merge($chatbot_atts, $atts); // $atts takes priority
		}

		if($mergeWithAll){
			$atts = TWXChatUtil::mergeChatbotAttributesWithAll($atts);
		}

		return $atts;
	}

	public static function mergeChatbotAttributesWithAll($atts){

		$allAtts = TWXChatUtil::getAllChatbotShortcodeAttributes();
		foreach($allAtts as $attName => $att){
			if(!isset($atts[$attName])){
				$atts[$attName] = isset($att['default_val']) ? $att['default_val'] : '';
			}
		}

		return $atts;
	}

	public static function isValidAvatarShortcodeAtts($atts){
		return isset($atts['avatar_character']) && !empty($atts['avatar_character']) && isset($atts['avatar_voice']) && !empty($atts['avatar_voice']);
	}

	public static function findPostsWithChatbotId($id) {
		global $wpdb;

    	$search = '[twxchat id="' . (int)$id;
    	$like = '%' . $wpdb->esc_like($search) . '%';

		// Search posts and pages where the post_content contains the shortcode
		//$like = '%' . $wpdb->esc_like('[' . $shortcode) . '%';

		$results = $wpdb->get_results(
			$wpdb->prepare("
				SELECT ID, post_title, post_type 
				FROM {$wpdb->posts}
				WHERE post_status IN ('publish','private') 
				AND post_type IN ('post', 'page')
				AND post_content LIKE %s
			", $like)
		);

		return $results;
	}

	public static function getFiles($apiKey, $purpose = 'assistants') {

		// Get all OpenAI files related to the API Key
		$url = "https://api.openai.com/v1/files";
		
		$params = [];
		if(!empty($purpose)) $params['purpose'] = $purpose;
		
		$response = self::getHttpOpenAI_WP($apiKey, $url, $params, 'GET');

		return $response['data'];	
	}
	
	public static function deleteFile($apiKey, $fileId){
		
		// Delete a file from OpenAI storage by its File ID (must be owned by the apiKey)
		$url = "https://api.openai.com/v1/files/$fileId";
		$response = TWXChatUtil::getHttpOpenAI_WP($apiKey, $url, null, 'DELETE');
		
		return $response;  // Deletion Status
	}

	public static function getAssistants($apiKey) {

		// Retrieves all Assistants connected to ApiKey
		$url = "https://api.openai.com/v1/assistants";
		
		$response = TWXChatUtil::getHttpOpenAI_WP($apiKey, $url, null, 'GET');
		if(isset($response['error'])){
			$ret['error']['message'] = $response['error']['message'];
			return $ret;
		}

		return $response['data'];
	}

	public static function deleteUnusedFiles($apiKey){
		
		// Deletes any Files in OpenAI Project associated with apiKey that are not in use (by Assistants/Vector Stores etc)

		TWXChatUtil::adminLog("TWXChatUtil::deleteUnusedFiles() start");
		
		$assistants = TWXChatUtil::getAssistants($apiKey);
		if(isset($assistants['error'])){
			TWXChatUtil::adminLog("TWXChatUtil::deleteUnusedFiles() error deleting unused spider files: ".$assistants['error']['message']);
			return false;
		}

		$inUseFileIds = [];
		$count = 0;
		
		foreach($assistants as $asst){
			if(isset($asst['tool_resources']['file_search']['vector_store_ids'])){
				$asstId = $asst['id'];
				$vsIds = $asst['tool_resources']['file_search']['vector_store_ids'];
				if(empty($vsIds)) continue;
				$assistObj = new TWXChatOpenAIAssistant($apiKey, $asstId);
				foreach($vsIds as $vsId){
					$vsFiles = $assistObj->getVectorStoreFiles($vsId);
					if(isset($vsFiles['error'])) continue;
					foreach($vsFiles as $vsFile){
						$inUseFileIds[$vsFile['id']] = true;
					}
				}
			}
		}

		$files = TWXChatUtil::getFiles($apiKey);

		if(is_array($files)){
			foreach($files as $file){
				$fileId = $file['id'];
				$fileString = $file['filename']." ($fileId)";

				//TWXChatUtil::log("deleteUnusedFiles:: check file $fileString");

				if(isset($inUseFileIds[$fileId])) {
					//TWXChatUtil::log("deleteUnusedFiles:: $fileString aready in use, skip");
					continue;  // dont delete in use file
				}
				
				//TWXChatUtil::log("deleteUnusedFiles:: delete unused file $fileString");
				
				$response = TWXChatUtil::deleteFile($apiKey, $fileId);
				if(isset($response['error']['message'])){
					TWXChatUtil::adminLog($response['error']['message']);
				}else{
					$count++;
				}
			}
		}

		return $count;
	}

	public static function appendQuestionMark($string) {
		// append ? to end of string if not already there - can be the difference between Knowledge Base entry match or not for a user query
		if (substr($string, -1) !== '?') {
			$string .= '?';
		}
		return $string;
	}

	public static function generateRandomFilename( $extension = '', $length = 10 ) {
		// Generate a unique string using wp_generate_password
		$filename = strtolower( wp_generate_password( $length, false, false ) );

		// Sanitize for safe use in file systems
		$filename = sanitize_file_name( $filename );

		// Append extension if given
		if ( $extension ) {
			$extension = ltrim( $extension, '.' ); // remove leading dot if present
			$filename .= '.' . $extension;
		}

		return $filename;
	}


	public static function makeDir($dir_path, $chmod = 0755) {
		if ( ! function_exists('WP_Filesystem') ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
		}

		global $wp_filesystem;

		if ( ! $wp_filesystem ) {
			WP_Filesystem();
		}

		$dir_path = untrailingslashit($dir_path);

		if ( $wp_filesystem->is_dir($dir_path) ) {
			return true;
		}

		$parent_dir = dirname($dir_path);
		if ( ! $wp_filesystem->is_dir($parent_dir) ) {
			// Recursively create parent directory
			self::makeDir($parent_dir, $chmod);
		}

		return $wp_filesystem->mkdir($dir_path, $chmod);
	}

	public static function removeDir( $dir_path, $recursive = false ) {
		global $wp_filesystem;

		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		// Check if the directory exists
		if ( $wp_filesystem->is_dir( $dir_path ) ) {
			return $wp_filesystem->rmdir( $dir_path, $recursive );
		}

		return false; // Directory did not exist or could not be removed
	}

	public static function touch($file_path) {
		global $wp_filesystem;

		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		// Create file if it doesn't exist
		if ( ! $wp_filesystem->exists( $file_path ) ) {
			$wp_filesystem->put_contents( $file_path, '', FS_CHMOD_FILE );
			return true;
		}

		// Update modified time by re-writing contents
		$contents = $wp_filesystem->get_contents( $file_path );
		return $wp_filesystem->put_contents( $file_path, $contents, FS_CHMOD_FILE );
	}

	public static function fileWrite( $file_path, $content, $append = false, $max_lines = null) {
		global $wp_filesystem;

		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		// If appending, read the current content
		if ( $append && $wp_filesystem->exists( $file_path ) ) {
			$existing = $wp_filesystem->get_contents( $file_path );
			$content = $existing . $content;
		}

		// If max_lines is set, trim to the last X lines
		if ( $max_lines !== null && is_numeric( $max_lines ) && $max_lines > 0 ) {
			$lines = explode( "\n", $content );
			if ( count( $lines ) > $max_lines ) {
				$lines = array_slice( $lines, -$max_lines ); // Keep last X lines
			}
			$content = implode( "\n", $lines );
		}

		return $wp_filesystem->put_contents( $file_path, $content, FS_CHMOD_FILE );
	}

	public static function recursiveCopy($src, $dst) {
		$dir = opendir($src);
		TWXChatUtil::makeDir($dst);

		while(false !== ($file = readdir($dir))) {
			if (($file !== '.') && ($file !== '..')) {
				$srcPath = $src . '/' . $file;
				$dstPath = $dst . '/' . $file;

				if (is_dir($srcPath)) {
					TWXChatUtil::recursiveCopy($srcPath, $dstPath);
				} else {
					copy($srcPath, $dstPath);
				}
			}
		}
		closedir($dir);
	}

	public static function recursiveDelete($dir_path){
		// Make sure WP_Filesystem is loaded
		if ( ! function_exists('WP_Filesystem') ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
		}

		global $wp_filesystem;

		// Initialize the filesystem if not already done
		if ( ! $wp_filesystem ) {
			WP_Filesystem();
		}

		// Normalize and validate path
		$dir_path = trailingslashit($dir_path);

		if ( ! $wp_filesystem->is_dir($dir_path) ) {
			return false; // Not a directory
		}

		// Attempt to delete directory recursively
		return $wp_filesystem->delete($dir_path, true, 'd'); // recursive, type = 'd' for directory
	}

	public static function updateOption($key,$val){
		// Needs access to WP context
		if(empty($key)) return false;
		try {
			$options = get_option('twx_chat_options', []);
			$options[$key] = $val;
			update_option('twx_chat_options', $options);
			return true;
		}catch(Exception $ex){
			TWXChatUtil::adminLog("TWXChatUtil::updateOption() Error updating Option '$key' to '$val'. ".$ex->getMessage(),'a',true);
			return false;
		}
	}

	public static function log($message, $mode = 'a', $isError = false) {
		
		// Main Log
		try {
			$dir = TWXChatUtil::getUploadsDir('logs');
			if(!file_exists($dir)) TWXChatUtil::makeDir($dir);
			$logfile = $dir.'twx_chat.log';
			$logEntry = current_time('mysql') . " - " . $message . "\n";
			$append = ($mode === 'a');
			TWXChatUtil::fileWrite($logfile, $logEntry, $append, 1000);

			if($isError){
				if ( defined( 'WP_DEBUG' ) && WP_DEBUG && defined( 'WP_DEBUG_LOG' ) && WP_DEBUG_LOG ) {
					error_log('[TWX-Chat Plugin] '. current_time('mysql') . " - " . $message);
				}
			}
		
		} catch(Exception $ex) {
			// Do nothing
		}
	}	

	public static function getChatLogText(){

		$file_path = TWXChatUtil::getUploadsDir('logs').'twx_chat.log';

		global $wp_filesystem;
		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		if(!$wp_filesystem->exists($file_path)) return '';

		return $wp_filesystem->get_contents($file_path);
	}

	public static function adminLog($message, $mode = 'a', $isError = false) {
		
		// Main Log
		try {
			$dir = TWXChatUtil::getUploadsDir('logs');
			if(!file_exists($dir)) TWXChatUtil::makeDir($dir);
			$logfile = $dir.'twx_chat_admin.log';
			$logEntry = current_time('mysql') . " - " . $message . "\n";
			$append = ($mode === 'a');
			TWXChatUtil::fileWrite($logfile, $logEntry, $append, 1000);

			if($isError){
				if ( defined( 'WP_DEBUG' ) && WP_DEBUG && defined( 'WP_DEBUG_LOG' ) && WP_DEBUG_LOG ) {
					error_log('[TWX-Chat Plugin] '. current_time('mysql') . " - " . $message);
				}
			}
		
		} catch(Exception $ex) {
			// Do nothing
		}
	}

	public static function getAdminLogText(){

		$file_path = TWXChatUtil::getUploadsDir('logs').'twx_chat_admin.log';

		global $wp_filesystem;
		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		if(!$wp_filesystem->exists($file_path)) return '';

		return $wp_filesystem->get_contents($file_path);
	}

	public static function consoleLog($message) {

		try {
			$dir = TWXChatUtil::getUploadsDir('logs');
			if(!file_exists($dir)) TWXChatUtil::makeDir($dir);
			$date = current_time('Y-m-d'); // e.g., 2025-04-16
			$logFile = $dir."{$date}-console.log";
		
			$logLine = $message . PHP_EOL;
		
			// Open the file and lock it - Need locking thus can't use WP Filesystem methods
			$fp = fopen($logFile, 'a');
			if ($fp) {
				// Exclusive lock (waits until it gets the lock)
				if (flock($fp, LOCK_EX)) {
					fwrite($fp, $logLine);
					fflush($fp); // ensure it's written
					flock($fp, LOCK_UN); // release lock
				}
				fclose($fp);
			} else {
				TWXChatUtil::log("TWXChatUtil::consoleLog() Failed to open log file: $logFile");
				return false;
			}
		
		} catch(Exception $ex) {
			// Do nothing
			return false;
		}
		return true;
	}	
	
}