<?php
/* Website Spider/Crawler Class 
Expects
	Wordpress context
	TWXChatUtil
	TWXChatKnowledgeBaseUtil
	TWXChatPro (optional)
	Vendor autoload: HtmlConverter
*/

class TWXChatSpider
{
	//public const RESULTS_FILENAME = 'twxchat-sitecrawl.md';

    // Class properties (module variables)
    private $_visitedUrls = [];
	private $_topDomain = '';

    // Constructor method
    public function __construct()
    {
        // Assign values to properties
    }

    // Destructor method
    public function __destruct()
    {
        // Log or perform actions needed at object destruction 
    }

	public static function runSiteCrawl($schedule) {

		// run site crawler now

		TWXChatSpider::log('TWXChatSpider start', 'w');
		TWXChatSpider::clearProgress();
		TWXChatSpider::appendProgress('Crawler Started...');

		try {
			
			$options = get_option('twx_chat_options');
			$openaiApiKey = $options['openai_apikey'] ?? '';
			$crawlPublicPosts = ($options['site_crawler_public_posts'] ?? '0') === '1';
			$crawlPrivatePosts = ($options['site_crawler_private_posts'] ?? '0') === '1';
			$crawlSiteMedia = ($options['site_crawler_onsite_media'] ?? '0') === '1';
			$crawlSitemap = ($options['site_crawler_sitemap'] ?? '0') === '1';
			$sitemapUrl = $options['site_crawler_start_url'] ?? '';
			$updateAssistant = ($options['site_crawler_update_assistant'] ?? '0') === '1';
			$assistantId = $options['site_crawler_assistant_id'] ?? '';
			$crawlGoogleDrive = ($options['site_crawler_google_drive'] ?? '0') === '1';
			$gdriveFolderId = $options['site_crawler_gdrive_folder_id'] ?? '';
			$googleApiKey = $options['google_apikey'] ?? '';

			TWXChatSpider::log('TWXChatSpider params retrieved');

			update_option('twx_chat_site_crawler_running', true);

			// Delete completed file if exists. This file can be monitored by ajax calls to detect crawl completed
			$spiderCompletedFile = TWXChatUtil::getUploadsDir('spider_results').'twx_spider_completed.txt';
			if(file_exists($spiderCompletedFile)) wp_delete_file($spiderCompletedFile);

			TWXChatSpider::appendProgress("Schedule: $schedule");
			
			$completeAfterScriptDone = true;
			$bigMarkdownFileName = 'twxchat-'.TWXChatUtil::generateRandomFilename('md');
			$bigMarkdownFilePath = TWXChatUtil::getUploadsDir('spider_results').$bigMarkdownFileName;
			if(file_exists($bigMarkdownFilePath)) wp_delete_file($bigMarkdownFilePath);

			if($crawlPublicPosts || $crawlPrivatePosts){
				$num_items = TWXChatSpider::crawlPosts($openaiApiKey, $crawlPublicPosts, $crawlPrivatePosts, $bigMarkdownFilePath);
				TWXChatSpider::appendProgress("$num_items changed onsite posts/pages refreshed in Knowledge Base");
			}

			if($crawlSiteMedia){
				$num_items = TWXChatSpider::crawlSiteMedia($openaiApiKey, $bigMarkdownFilePath);
				TWXChatSpider::appendProgress("$num_items onsite media items with descriptions/captions refreshed in Knowledge Base");
			}

			if($crawlGoogleDrive && !empty($gdriveFolderId) && !empty($googleApiKey)){
				if(TWXChatUtil::customerPlanAccess('pro') && class_exists('TWXChatPro')){
					$num_items = TWXChatPro::crawlGoogleFolder($googleApiKey, $gdriveFolderId, $updateAssistant, $assistantId, 'TWXChatSpider::appendProgress', $bigMarkdownFilePath);
					$assistant = ($updateAssistant && !empty($assistantId)) ? " and Updated to OpenAI Assistant ID '$assistantId'" : '';
					TWXChatSpider::appendProgress("$num_items files in Google Drive Folder '$gdriveFolderId' loaded/refreshed in Knowledge Base".$assistant);
				}else{
					TWXChatSpider::log('Unauthorized access to Google Drive Folder Crawl feature');
				}
			}

			if($crawlSitemap && !empty($sitemapUrl)){

				if(!TWXChatUtil::urlExists($sitemapUrl)){
					TWXChatSpider::appendProgress("$sitemapUrl does not exist. Aborting sitemap crawl");
				}else{
					if(TWXChatUtil::customerPlanAccess('pro') && class_exists('TWXChatPro')){
						list($started, $numUrls, $numBatches, $message) = TWXChatSpider::crawlSitemapInBackground($sitemapUrl, $updateAssistant, $assistantId);
						if(!$started){
							TWXChatSpider::appendProgress($message);
						}else{
							// Leave twx_chat_site_crawler_running option as true until batches complete
							$completeAfterScriptDone = false;
						}
					}else{
						TWXChatSpider::log('Unauthorized access to Sitemap Crawler feature');
					}
				}
			}

			if($completeAfterScriptDone){
				$bigMarkdownFileUrl = TWXChatUtil::getUploadsUrl('spider_results', $bigMarkdownFileName);
				TWXChatSpider::appendProgress('Site Crawler Markdown File Generated. <a href="'.$bigMarkdownFileUrl.'" download="'.$bigMarkdownFileName.'">Download here</a>');
				TWXChatSpider::appendProgress("Completed Crawl");
				update_option('twx_chat_site_crawler_running', false);
			}


		}catch(Exception $ex){
			
			TWXChatSpider::log($ex->getMessage());
			TWXChatSpider::appendProgress('Error occurred: '.$ex->getMessage());
			update_option('twx_chat_site_crawler_running', false);

		}
	}

	public static function crawlPosts($openaiApiKey, $include_public, $include_private, $big_markdown_file_path = '') {

		// Crawl onsite Posts/Pages

		if(!$include_public && !$include_private) return 0;

		$hcParams = [ 'strip_tags' => true];
		try{
			$converter = new League\HTMLToMarkdown\HtmlConverter($hcParams); // from vendor autoload
		}catch(Exception $ex){
			TWXChatSpider::log("twx_chat_crawlPosts:: Error creating HtmlConverter: ".$ex->getMessage());
			return 0;
		}
		
		$num_loaded = 0;
		$post_statuses = [];
		if($include_public) $post_statuses[] = 'publish';
		if($include_private) $post_statuses[] = 'private';

		$args = [
			'post_type'      => ['post', 'page'],
			'post_status'    => $post_statuses,
			'posts_per_page' => -1,
		];

		$query = new WP_Query($args);
		$posts = $query->posts;

		foreach ($posts as $post) {
			setup_postdata($post);

			// Get basic fields
			$post_id    = $post->ID;
			$title      = get_the_title($post_id);
			$html    	= apply_filters('the_content', $post->post_content); // Rendered HTML content
			$excerpt    = get_the_excerpt($post_id);
			$date       = get_the_date('', $post_id);
			$modified   = get_the_modified_date('', $post_id);
			$source  = get_permalink($post_id);

			// Get SEO metadata (from Yoast or other plugins if present)
			$meta_description = get_post_meta($post_id, '_yoast_wpseo_metadesc', true);
			$meta_keywords    = get_post_meta($post_id, 'keywords', true); // Custom field

			// Fallback if no Yoast description
			if (!$meta_description && $excerpt) {
				$meta_description = $excerpt;
			}

			if(empty($html)) continue;

			$markdown = TWXChatSpider::getMarkdown($converter, $html);
			$text = (!empty($meta_description) ? "description: $meta_description\n" : '');
			$text .= (!empty($meta_keywords) ? "keywords: $meta_keywords\n" : '');
			$text .= $markdown;

			try {

				if(TWXChatKnowledgeBaseUtil::hasOrigTextChanged($source, $text)){
					TWXChatSpider::log("load post/page into Knowledge Base: $source");
					TWXChatKnowledgeBaseUtil::storeEntry($openaiApiKey, $source, $title, $text, true);
					$num_loaded++;
				}else{
					TWXChatSpider::log("no change to source: $source");
				}

				if(!empty($big_markdown_file_path)) TWXChatUtil::fileWrite($big_markdown_file_path, "source: $source\ntitle: $title\n".$text."\n\n", true);

			} catch (Exception $ex) {
				TWXChatSpider::log("twx_chat_crawlPosts:: Error ".$ex->getMessage());
			}
		}

		// Check if dynamic front page and if so, inlcude
		$response = wp_remote_get( home_url('/') );

		if ( ! is_wp_error($response) ) {
			$html = wp_remote_retrieve_body($response);

			if ( ! empty($html) ) {
				$markdown = TWXChatSpider::getMarkdown($converter, $html);
				$source = home_url('/');
				$title = 'Homepage';
				$text = $markdown;

				if ( TWXChatKnowledgeBaseUtil::hasOrigTextChanged($source, $text) ) {
					TWXChatSpider::log("Crawled and stored rendered homepage: $source");
					TWXChatKnowledgeBaseUtil::storeEntry($openaiApiKey, $source, $title, $text, true);
					$num_loaded++;
				} else {
					TWXChatSpider::log("No changes detected in rendered homepage.");
				}
			} else {
				TWXChatSpider::log("Empty HTML body returned from homepage.");
			}
		} else {
			TWXChatSpider::log("Error fetching homepage: " . $response->get_error_message());
		}

		wp_reset_postdata();

		return $num_loaded;
	}

	public static function crawlSiteMedia($openaiApiKey, $big_markdown_file_path = '') {

		$num_loaded = 0;

		$args = [
			'post_type'      => 'attachment',
			'post_status'    => 'inherit',
			'posts_per_page' => -1, // or a number like 20
			'post_mime_type' => 'image', // or 'audio', 'video', or leave out for all types
		];

		$media_query = new WP_Query($args);
		if ($media_query->have_posts()) {
			while ($media_query->have_posts()) {
				$media_query->the_post();

				$id = get_the_ID();
				$keywords = isset(wp_get_attachment_metadata($id)['image_meta']['keywords']) && !empty(wp_get_attachment_metadata($id)['image_meta']['keywords']) ? json_encode(wp_get_attachment_metadata($id)['image_meta']['keywords']) : null;
				$meta = [
					'id'=>$id,
					'title'=>get_the_title(),
					'caption'=>wp_get_attachment_caption($id),
					'description'=>get_the_content(),
					'alt_text'=>get_post_meta($id, '_wp_attachment_image_alt', true),
					'mime_type'=>get_post_mime_type(),
					'file_url'=>wp_get_attachment_url(),
					'meta_data'=>wp_get_attachment_metadata($id),
					'keywords'=>$keywords,
					'uploaded_date'=>get_the_date(),
				];

				if(empty($meta['description']) && empty($meta['caption']) && empty($meta['alt_text'])) continue;

				$source = $meta['file_url'] ?? "media_{$id}";
				$title = $meta['title'] ?? null;
				$text = (!empty($meta['caption']) ? "caption: {$meta['caption']}\n" : "")
					. (!empty($meta['description']) ? "description: {$meta['description']}\n" : "") 
					. (!empty($meta['alt_text']) ? "alt_text: {$meta['alt_text']}" : "");
				TWXChatSpider::log("load media item into Knowledge Base: $source");
				try {
					if(TWXChatKnowledgeBaseUtil::hasOrigTextChanged($source, $text)){
						TWXChatKnowledgeBaseUtil::storeEntry($openaiApiKey, $source, $title, $text, true);
						$num_loaded++;
					}else{
						TWXChatSpider::log("no change to source: $source");
					}

					if(!empty($big_markdown_file_path)) TWXChatUtil::fileWrite($big_markdown_file_path, "source: $source\ntitle: $title\n".$text."\n\n", true);

				} catch (Exception $ex) {
					TWXChatSpider::log("error ".$ex->getMessage());
				}
			}
			wp_reset_postdata();
		} else {
			//echo "No media items found.";
		}

		return $num_loaded;

	}

	public static function crawlSitemapInBackground($sitemapUrl, $updateAssistant, $assistantId){

		//$path = plugin_dir_path(__FILE__) . 'spider_batch.php';
		$path = plugin_dir_path(dirname(__FILE__)) . 'spider_batch.php';
		if(!file_exists($path)){
			return [false, 0, 0, 'Background processor file not available'];
		}

		$batchesOf = 35;
		$spider = new TWXChatSpider();
		$token = get_option('twx_chat_server_token','');
		
		TWXChatSpider::appendProgress('Crawl Sitemap: '.$sitemapUrl);
		$urlsToCrawl = TWXChatPro::getSitemapUrlsToCrawl($sitemapUrl, $spider);
		if(empty($urlsToCrawl)){
			return [false, 0, 0, 'No valid URLs in sitemap to crawl'];
		}
		TWXChatSpider::appendProgress('Found '.count($urlsToCrawl).' unique URLs');

		$urlBatches = array_chunk($urlsToCrawl, $batchesOf);
		foreach ($urlBatches as $index => $urls) {

			$batchNo = ($index+1);
			$params = [ 
				'assistantId' => ($updateAssistant && $assistantId) ? $assistantId : '',
				'batchNo' => $batchNo,
				'numBatches' => count($urlBatches),
				'urls' => $urls,
				'token' => $token,
				'nocache' => substr(md5(uniqid(wp_rand(), true)), 0, 8),
			];
			
			//$url = plugins_url('spider_batch.php', __FILE__);
			$url = plugins_url('spider_batch.php', dirname(__FILE__));

			// Start Batch Crawl in Background
			TWXChatSpider::log("twx_chat_executeBackground batch no $batchNo @ $url");
			TWXChatPro::executeBackground($url, $params);
		}

		return [true, count($urlsToCrawl), count($urlBatches), 'Go!'];
	}


	public function getUrlsToCrawl($url){
		
		// inspect a url's contents and extract links (urls) within it
		// url could be a sitemap for instance
		
		try {
			$html = $this->fetchUrlContent($url);
			if($html === false){
				TWXChatSpider::log("Unable to get content from url: $url");
				return [];
			}

			if($this->isSitemapContent($html)){
				$links = $this->getSitemapLinks($html);
				if(!empty($links) && $this->isSitemapUrl($links[0])) {
					// Sitemap contains links to other Sitemaps
					TWXChatSpider::log("Sitemap $url contains links to other sitemaps. Traversing.");
					$nonSiteMapLinks = [];
					foreach($links as $link){
						$nonSiteMapLinks = array_merge($nonSiteMapLinks, $this->getUrlsToCrawl($link));
					}
					$links = $nonSiteMapLinks;
				}
			}else{
				$links = $this->extractLinksFromHtml($html, $url);
			}
			
			return $links;
			
		}catch(Exception $ex){
			TWXChatSpider::log("Error getting urls to crawl: " . $ex->getMessage());
			return [];
		}
		
	}
	
	public function crawlUrlsNonRecursive($urls, $batchNo, $createFiles, callable $progressCallback){
		
		$this->resetVars();
		
		// Crawl array of Urls, non recursivley
		$preserveLinks = true;
		$allowOffsite = true; // as we are looping through a list of user defined urls
		$i = 0;
		foreach ($urls as $url) {
			if($this->crawlUrl($url, 0, 0, $allowOffsite, $batchNo, $preserveLinks, $createFiles)) {
				$i++;
				$progressCallback($batchNo, count($urls), $i);
			}
		}
		
	}
	
	public function isSitemapUrl($url) {
		
		$html = $this->fetchUrlContent($url);
		if($html === false){
			throw new Exception("Unable to get content from url: $url");
		}
		
		return $this->isSitemapContent($html);
	}
	
	public function startCrawl($topDomain, $startUrl, $maxDepth, $allowOffsite = false, $createFiles = true, $mustMatch = null){
		
		// Begin recursive crawl
		
		$this->resetVars();
		
		// causes issues if not a dir eg: sitemap.xml
		//$startUrl = rtrim($startUrl, '/') . '/'; // ensure last char is a /
		
		if(empty($topDomain)){
			$this->_topDomain = wp_parse_url($startUrl)['host'] ?? '';
		}else{
			$this->_topDomain = $topDomain;
		}
		
		try {
			if($createFiles) {
				$dir = TWXChatUtil::getUploadsDir('spider_results',false);
				$this->removeAllSubdirsAndFiles($dir, ['twxchatspider_progress.txt']);
			} 
		} catch (Exception $e) {
			TWXChatSpider::log("Error removing previous spider_results: " . $e->getMessage());
			return false;
		}

		$batchNo = 0;
		$preserveLinks = true;
		$ret = $this->crawlUrl($startUrl, 0, $maxDepth, $allowOffsite, $batchNo, $preserveLinks, $createFiles, $mustMatch);
		
		TWXChatSpider::log('crawl done');
		
		return $ret;
	}
	
	public function removeAllSubdirsAndFiles($dir, $exceptFiles = [])
	{
		if (!is_dir($dir)) {
			throw new InvalidArgumentException(esc_html($dir) . ' must be a directory');
		}

		$items = scandir($dir);

		foreach ($items as $item) {
			if ($item === '.' || $item === '..') {
				continue;
			}

			$path = $dir . DIRECTORY_SEPARATOR . $item;

			if (is_dir($path)) {
				// Unix-based systems
				if (strtoupper(substr(PHP_OS, 0, 3)) !== 'WIN') {
					exec("rm -rf " . escapeshellarg($path));
				} else {
					// Windows systems
					exec("rmdir /S /Q " . escapeshellarg($path));
				}
			} else {

				//TWXChatUtil::log("item: $item. exceptFiles: ".var_export($exceptFiles,true).'. remove = '.(is_file($path) && !in_array($item, $exceptFiles)));

				if(is_file($path) && !in_array($item, $exceptFiles)) {
					// Remove files
					wp_delete_file($path);
				}
			}
		}
	}
	
	public function removeAllFilesInDir($dir)
	{
		if (!is_dir($dir)) {
			throw new InvalidArgumentException(esc_html($dir) . ' must be a directory');
		}

		// Get all files in the directory
		$files = glob($dir . '/*');

		// Iterate over the files
		foreach ($files as $file) {
			// If it's a file, delete it
			if (is_file($file)) {
				wp_delete_file($file);
			}
		}
		
	}

	public function getVisitedUrls()
	{
		return $this->_visitedUrls;
	}
	
	public function zipDirectory($sourceDir, $zipFilePath) {
		
		TWXChatSpider::log("Zipping spider results in $sourceDir to $zipFilePath");

		if (!extension_loaded('zip') || !file_exists($sourceDir)) {
			return false;
		}

		$zip = new ZipArchive();
		if (!$zip->open($zipFilePath, ZipArchive::CREATE)) {
			return false;
		}

		$sourceDir = realpath($sourceDir);

		$iterator = new RecursiveDirectoryIterator($sourceDir, RecursiveDirectoryIterator::SKIP_DOTS);
		$files = new RecursiveIteratorIterator($iterator, RecursiveIteratorIterator::SELF_FIRST);

		foreach ($files as $file) {
			$filePath = realpath($file);
			TWXChatSpider::log("Zipping $filePath...");
			if (is_dir($filePath)) {
				$zip->addEmptyDir(str_replace($sourceDir . DIRECTORY_SEPARATOR, '', $filePath . DIRECTORY_SEPARATOR));
			} elseif (is_file($filePath)) {
				$zip->addFile($filePath, str_replace($sourceDir . DIRECTORY_SEPARATOR, '', $filePath));
			}
		}

		return $zip->close();
	}
	
	public static function extractMetaTags($html) {
		
		$metaTags = [];
		
		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();
		
		// Get all meta tags
		$metaElements = $dom->getElementsByTagName('meta');
		
		$names = ['keywords','description']; //, 'og:image'];

		// Loop through meta tags and extract their attributes
		foreach ($metaElements as $metaElement) {
			$nameAttribute = $metaElement->getAttribute('name');
			if (in_array($nameAttribute, $names)) {
				$metaTags[$nameAttribute] = $metaElement->getAttribute('content');
			}
			if(!isset($metaTags[$nameAttribute])){
				$propertyAttribute = $metaElement->getAttribute('property');
				if( in_array($propertyAttribute, $names)){
					$metaTags[$propertyAttribute] = $metaElement->getAttribute('content');
				}
			}
		}
		
		return $metaTags;
	}

	public static function extractTitle($html) {
		
		$title = '';
		
		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();
		
		$titleElements = $dom->getElementsByTagName('title');
		if(!empty($titleElements)) {
			$title = $titleElements->item(0)->nodeValue;
		}
		
		return $title;
	}
	
	public static function getMarkdown(&$converter, $html){
		
		$html = self::stripDownHtml($html);
		
		$markdown = $converter->convert($html);

		$markdown = preg_replace('/(?<!\!)\[\s*\]\((.*?)\)/', '', $markdown); // Only remove links that have truly empty link text (not images)

		$markdown = preg_replace('/^\s+/m', '', $markdown); // remove leading spaces
		
		$markdown = str_replace("javascript:void(0);", "", $markdown);
		$markdown = str_replace("javascript:void(0)", "", $markdown);

		$markdown = str_replace("   \n", "\n", $markdown);
		$markdown = str_replace("  \n", "\n", $markdown);
		$markdown = str_replace(" \n", "\n", $markdown);
		$markdown = preg_replace('/(\r?\n){2,}/', "\n", $markdown); // turn multiple line breaks into single

		$markdown = preg_replace('/^-$/m', '', $markdown); // replace empty line items (-) with single line break
		$markdown = preg_replace('/(\r?\n){2,}/', "\n", $markdown); // turn multiple line breaks into single (again)

		return $markdown;
	}
	
	public static function removeUniformColumns(array $array) {
		
		if (empty($array)) {
			return $array;
		}

		// Extract column names
		$columns = array_keys(reset($array));
		$columnsToRemove = [];

		// Check each column
		foreach ($columns as $column) {
			$firstValue = null;
			$allSame = true;

			foreach ($array as $row) {
				if ($firstValue === null) {
					$firstValue = $row[$column];
				} elseif (isset($row[$column]) && $row[$column] !== $firstValue) {
					$allSame = false;
					break;
				}
			}

			if ($allSame) {
				$columnsToRemove[] = $column;
			}
		}

		// Remove uniform columns while preserving keys
		foreach ($array as $key => &$row) {
			foreach ($columnsToRemove as $column) {
				unset($row[$column]);
			}
		}

		return $array;
	}
	
	public static function getCommonHeadersAndFooters($markdowns) {
		
		$headers = [];
		$footers = [];

		// Extract all potential headers and footers
		foreach ($markdowns as $markdown) {
			$lines = explode("\n", $markdown);
			$headers[] = self::extractHeader($lines);
			$footers[] = self::extractFooter($lines);
		}
		
		// Identify the common headers and footers
		$commonHeaderLines = self::findCommonLines($headers);
		$commonFooterLines = self::findCommonLines($footers);
		if (!empty($commonFooterLines)) $commonFooterLines = array_reverse($commonFooterLines);
		//if(!empty($commonFooter)) krsort($commonFooter); // need to revert to original sort order for footer lines

		/*
		return [
			'header' => implode("\n", $commonHeader),
			'footer' => implode("\n", $commonFooter)
		];
		*/
		return [
			'headerLines' => $commonHeaderLines,
			'footerLines' => $commonFooterLines,
		];
	}

	///////////////////// PRIVATES ////////////////////////

	private function resetVars(){
		
		$this->_visitedUrls = [];
		$this->_topDomain = '';
	}

	private function crawlUrl($url, $currentDepth, $maxDepth = 2, $allowOffsite = false, $batchNo = 1, $preserveLinks = true, $createFiles = true, $mustMatch = null){
		
		// Main function. Recursive crawl a url and its url links until maxDepth reached
		// If $createFiles is false, no files are generated but the visited links will still increment

		TWXChatSpider::log("crawl batch $batchNo, depth $currentDepth maxDepth $maxDepth : $url");
		
		// Check if URL has already been visited to avoid duplicates
		if (isset($this->_visitedUrls[$url])) {
			TWXChatSpider::log('already visited. skipping');
			return true;
		}
		
		if(!$allowOffsite){
			if(wp_parse_url($url)['host'] != $this->_topDomain) {
				TWXChatSpider::log('url offsite. skipping');
				return true;
			}
		}

		try {
			// Fetch HTML content from URL
			$html = $this->fetchUrlContent($url);
			if ($html === false) {
				throw new Exception("Failed to fetch HTML content for URL: {$url}");
			}
			
			$isSiteMap = false;
			if($this->isSitemapContent($html)){
				$isSiteMap = true;
				TWXChatSpider::log("is sitemap. not saving but will get links to crawl");
			}

			if(!$isSiteMap && $mustMatch !== null){
				if(strpos($url, $mustMatch) === false){
					TWXChatSpider::log("url does not match '$mustMatch'. skipping");
					return true;
				}
			}

			if(!$isSiteMap){
				if($createFiles){
					$filepath = TWXChatUtil::getUploadsDir('spider_results') . wp_parse_url($url, PHP_URL_HOST) . wp_parse_url($url, PHP_URL_PATH) . '.html';
					$this->saveContentToFile($html, $filepath);
					$this->_visitedUrls[$url] = $filepath;
				}else{
					$this->_visitedUrls[$url] = null;
				}
			}

			// Recursive crawl if current depth is less than max depth
			if ($currentDepth < $maxDepth) {
				$links = [];
				if($isSiteMap){
					$links = $this->getSitemapLinks($html, $mustMatch);
				}else{
					$links = $this->extractLinksFromHtml($html, $url, $mustMatch);
				}
				TWXChatSpider::log(count($links)." links found in page. traversing.");
				
				foreach ($links as $link) {
					$this->crawlUrl($link, ($currentDepth + 1), $maxDepth, $allowOffsite, $batchNo, $preserveLinks, $createFiles, $mustMatch);
				}
			}
		} catch (Exception $e) {
			TWXChatSpider::log("Error crawling URL {$url}: " . $e->getMessage());
			return false;
		}
		
		//TWXChatSpider::log('crawl finished for '.$url);
		
		return true;
	}
	

	private static function extractHeader(array $lines) {
		// Extract first few lines, assuming headers are usually at the start
		return array_slice($lines, 0, min(100, count($lines)));
	}

	private static function extractFooter(array $lines) {
		krsort($lines); // turn upside down to allow common lines check from bottom-up
		return array_slice($lines, 0, min(100, count($lines)));
	}

	private static function findCommonLines(array $lineArrays, int $minOccurrences = 2): array {
		$lineCount = [];

		foreach ($lineArrays as $lines) {
			$seen = [];
			foreach ($lines as $line) {
				$line = trim($line);
				if ($line === '') continue;
				// Avoid double counting within the same document
				if (!isset($seen[$line])) {
					$lineCount[$line] = ($lineCount[$line] ?? 0) + 1;
					$seen[$line] = true;
				}
			}
		}

		// Keep lines appearing at least $minOccurrences times
		$commonLines = [];
		foreach ($lineCount as $line => $count) {
			if ($count >= $minOccurrences) {
				$commonLines[] = $line;
			}
		}

		return $commonLines;
	}
	
	private static function findShortestArrayIndex($arrayOfArrays) {
		
		$minLength = PHP_INT_MAX;
		$shortestArrayIndex = null;

		foreach ($arrayOfArrays as $key => $subArray) {
			if (is_array($subArray)) {
				$currentLength = count($subArray);
				if ($currentLength < $minLength) {
					$minLength = $currentLength;
					$shortestArrayIndex = $key;
				}
			}
		}

		return $shortestArrayIndex;
	}

	/*
	private function fetchUrlContent($url, &$retMsg)
	{
		if(!ini_get('allow_url_fopen')){
			return $this->fetchUrlContentWP($url);
		}
		
		// Suppress errors for failed fetches
		$html = @file_get_contents($url);
		
		// Check if the fetch was successful
		if ($html === false) {
			// Log the error
			$error = error_get_last();
			$retMsg = "Error fetching URL $url. Error was: ".$error['message'];
			TWXChatSpider::log($retMsg);
			return false;
		}
		
		return $html;
	}
	*/
	
	private function fetchUrlContent($url) {
		// Use WordPress HTTP API
		$response = wp_remote_get($url, [
			'timeout' => 30,
			'redirection' => 5, // max redirects
			'sslverify' => true,
		]);

		if (is_wp_error($response)) {
			// Log the error
			$error_message = $response->get_error_message();
			TWXChatSpider::log("fetchUrlContent() error for url {$url}: {$error_message}");
			return false;
		}

		$http_status = wp_remote_retrieve_response_code($response);

		if ($http_status != 200) {
			TWXChatSpider::log("HTTP error: {$http_status} - Failed to fetch URL: {$url}");
			return false;
		}

		$body = wp_remote_retrieve_body($response);

		return $body;
	}

	/*
	private function fetchUrlContentCURL($url)
	{
		// Initialize cURL session - Using CURL instead of wp_remote_get() as this Class may be used in other Non-WP Projects
		$ch = curl_init();
		
		// Set cURL options
		curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
		curl_setopt($ch, CURLOPT_TIMEOUT, 30);

		// Execute cURL request
		$html = curl_exec($ch);
		
		// Check for cURL errors
		if (curl_errno($ch)) {
			$error_msg = curl_error($ch);
			TWXChatSpider::log('fetchUrlContentCURL error for url '.$url.': '. $error_msg);
			curl_close($ch);
			return false;
		}
		
		// Get HTTP status code
		$http_status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
		
		// Close cURL session
		curl_close($ch);
		
		// Check if the request was successful
		if ($http_status != 200) {
			TWXChatSpider::log('HTTP error: ' . $http_status . ' - Failed to fetch URL: ' . $url);
			return false;
		}
		
		return $html;
	}
	*/
	
	private function parseHtmlContent($html)
	{
		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();

		$xpath = new DOMXPath($dom);

		// Remove script, style, and other non-visible elements
		foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg | //img') as $element) {
			$element->parentNode->removeChild($element);
		}

		// Extract only visible content
		/*
		$mainContent = '';
		$body = $dom->getElementsByTagName('body')->item(0);
		if ($body) {
			foreach ($body->childNodes as $node) {
				if ($node->nodeType == XML_ELEMENT_NODE) {
					$mainContent .= $dom->saveHTML($node);
				} elseif ($node->nodeType == XML_TEXT_NODE) {
					$mainContent .= htmlspecialchars($node->nodeValue, ENT_QUOTES | ENT_HTML5, 'UTF-8');
				}
			}
		}
		*/

		$mainContent = '';
		foreach ($dom->getElementsByTagName('body')->item(0)->childNodes as $node) {
			$mainContent .= $dom->saveHTML($node);
		}

		return $mainContent;
	}
	
	private static function stripDownHtml($html){
		
		if(empty($html)) return '';

		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();
		
		$xpath = new DOMXPath($dom);

		// Remove script, style, and other non-visible elements
		//foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg | //img') as $element) {
		foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg') as $element) {
			$element->parentNode->removeChild($element);
		}
		
		$body = $dom->getElementsByTagName('body')->item(0);
		if ($body) {
			$innerHtml = '';
			foreach ($body->childNodes as $child) {
				$innerHtml .= $dom->saveHTML($child);
			}
			return $innerHtml;
		}

		return '';
	}
	
	private function hasAllSameValues($arr) {
		if (empty($arr)) {
			return true; // Consider an empty array as having all the same values
		}

		$firstValue = $arr[0];
		foreach ($arr as $value) {
			if ($value !== $firstValue) {
				return false;
			}
		}

		return true;
	}
	
	private function getTextContent($html, $preserveLinks){
		
		// returns only Text contents of given html
		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();
		
		try{
			$xpath = new DOMXPath($dom);

			// Remove script, style, and other non-visible elements
			foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg | //img') as $element) {
				$element->parentNode->removeChild($element);
			}
			
			$mainContent = '';
			$body = $dom->getElementsByTagName('body')->item(0);

			if ($body) {
				$mainContent = $this->getTextFromNode($body, $preserveLinks);
			}else{
				TWXChatSpider::log("could not retrieve body of html");
			}
		
			// remove whitespace
			$mainContent = trim(preg_replace('/\s+/', ' ', $mainContent));
			
			return $mainContent;
			
		}catch(Exception $ex){
			return null;
		}
	}
	
	private function getTextFromNode($node, $preserveLinks = false) {
		
		// Recursive get inner text of a node
		
		$textContent = '';
		foreach ($node->childNodes as $child) {
			if ($child->nodeType == XML_ELEMENT_NODE) {
				if ($child->nodeName == 'br') {
					$textContent .= ' ';
				} elseif ($child->nodeName == 'a' && $preserveLinks) {
					// Recursively process the anchor's child nodes
					$innerText = trim($this->getTextFromNode($child, $preserveLinks));
					if(!$this->nonContent($innerText)){
					//if(!empty($innerText)){
						// Append the opening anchor tag with its href attribute
						$href = $child->getAttribute('href');
						$textContent .= ' <a href="' . htmlspecialchars($href, ENT_QUOTES) . '">';
						$textContent .= $innerText;
						// Append the closing anchor tag
						$textContent .= '</a> ';
					}
				} elseif ($child->nodeName == 'li') {
					// Keep li elements otherwise openai can seperate/merge text unexpectedly
					$innerText = trim($this->getTextFromNode($child, $preserveLinks));
					if(!$this->nonContent($innerText)) $textContent .= '<li>'.$innerText.'</li>';
					//if(!empty($innerText)) $textContent .= '<li>'.$innerText.'</li>';
				} else {
					$textContent .= ' ' . $this->getTextFromNode($child, $preserveLinks);
				}
			} elseif ($child->nodeType == XML_TEXT_NODE) {
				$textContent .= ' ' . $child->nodeValue;
			}
		}
		
		return $textContent;
	}
	
	private function nonContent($text){

		if(empty(trim($text))) return true;
		
		$non = ['Home','|'];
		if(in_array(trim($text), $non)) return true;
		return false;
	}

	// Function to save content to file
	private function saveContentToFile($content, $filename)
	{
		$directory = dirname($filename);
		if (!is_dir($directory)) {
			TWXChatUtil::makeDir($directory);
		}
		file_put_contents($filename, $content);
	}

	
	// Function to extract links from HTML content
	private function extractLinksFromHtml($html, $baseUrl, $mustMatch = null)
	{
		$links = [];

		try{
			$dom = new DOMDocument();
			libxml_use_internal_errors(true); // Suppress HTML parsing errors
			$dom->loadHTML($html);
			libxml_clear_errors();

			$baseHost = wp_parse_url($baseUrl, PHP_URL_HOST);
			$baseScheme = wp_parse_url($baseUrl, PHP_URL_SCHEME);

			$anchorTags = $dom->getElementsByTagName('a');
			foreach ($anchorTags as $tag) {
				$href = $tag->getAttribute('href');
				if ($href && strpos($href, '#') !== 0) { // Exclude anchor links
					$absoluteUrl = $this->resolveUrl($baseScheme, $baseHost, $baseUrl, $href);
					if ($absoluteUrl) {
						if($mustMatch !== null && strpos($absoluteUrl, $mustMatch) === false) continue;
						$links[] = $absoluteUrl;
					}
				}
			}
		}catch(Exception $ex){
			TWXChatSpider::log($ex->getMessage());
		}

		return $links;
	}

	private function resolveUrl($baseScheme, $baseHost, $baseUrl, $href)
	{
		if (strpos($href, 'http') === 0) {
			return $href; // Absolute URL
		} elseif (strpos($href, '/') === 0) {
			return "$baseScheme://$baseHost$href"; // Root-relative URL
		} else {
			$basePath = rtrim(wp_parse_url($baseUrl, PHP_URL_PATH), '/') . '/';
			return "$baseScheme://$baseHost/$basePath$href"; // Relative URL
		}
	}

	private function removeAllSubdirs($dir)
	{
		if (!is_dir($dir)) {
			throw new InvalidArgumentException(esc_html($dir) . ' must be a directory');
		}

		$items = scandir($dir);

		foreach ($items as $item) {
			if ($item === '.' || $item === '..') {
				continue;
			}

			$path = $dir . DIRECTORY_SEPARATOR . $item;

			if (is_dir($path)) {
				// Unix-based systems
				if (strtoupper(substr(PHP_OS, 0, 3)) !== 'WIN') {
					exec("rm -rf " . escapeshellarg($path));
				} else {
					// Windows systems
					exec("rmdir /S /Q " . escapeshellarg($path));
				}
			}
		}
	}

	private function isSitemapContent($content) {
		// Check if the content is XML
		libxml_use_internal_errors(true);
		$xml = simplexml_load_string($content);
        libxml_clear_errors();

		if ($xml === false) {
			return false; // Not valid XML
		}

		// Check for common sitemap attributes and elements
		$rootName = $xml->getName();
		$namespaces = $xml->getNamespaces(true);

		// Sitemap namespaces to look for
		$sitemapNamespaces = [
			'http://www.sitemaps.org/schemas/sitemap/0.9',
			'http://www.google.com/schemas/sitemap/0.9'
		];

		// Check if the root element is 'urlset' or 'sitemapindex' and if namespaces match
		if($rootName == 'urlset' || $rootName == 'sitemapindex') return true;
		if(isset($namespaces['']) && in_array($namespaces[''], $sitemapNamespaces)) return true;

		return false;
	}

	private function display_xml_error($error, $xml) {
		$return = $xml[$error->line - 1] . "\n";
		$return .= str_repeat('-', $error->column) . "^\n";

		switch ($error->level) {
			case LIBXML_ERR_WARNING:
				$return .= "Warning $error->code: ";
				break;
			case LIBXML_ERR_ERROR:
				$return .= "Error $error->code: ";
				break;
			case LIBXML_ERR_FATAL:
				$return .= "Fatal Error $error->code: ";
				break;
		}

		$return .= trim($error->message) .
				   "\n  Line: $error->line" .
				   "\n  Column: $error->column";

		if ($error->file) {
			$return .= "\n  File: $error->file";
		}

		return "$return\n\n--------------------------------------------\n\n";
	}
	
	private function getSitemapLinks($content, $mustMatch = null) {
		// Check if the content is XML
		libxml_use_internal_errors(true);
		$xml = simplexml_load_string($content);
		libxml_clear_errors();

		// Initialize an empty array to store the links
		$links = [];

		// Check if the XML is valid
		if ($xml !== false) {
			$rootName = $xml->getName();

			// If the root element is 'urlset', extract links from 'url' elements
			if ($rootName == 'urlset') {
				foreach ($xml->url as $url) {
					$link = (string) $url->loc;
					if($mustMatch !== null && strpos($link, $mustMatch) === false) continue;
					$links[] = $link;
				}
			}
			// If the root element is 'sitemapindex', extract links from 'sitemap' elements
			elseif ($rootName == 'sitemapindex') {
				foreach ($xml->sitemap as $sitemap) {
					$links[] = (string) $sitemap->loc;
				}
			}
		}

		return $links;
	}

	public static function log($message, $mode = 'a') {
		try {
			$dir = TWXChatUtil::getUploadsDir('logs');
			TWXChatUtil::makeDir($dir);
			$logfile = $dir.'twx_spider.log';
			$logEntry = current_time('mysql') . " - " . $message . "\n";
			$append = ($mode === 'a');
			TWXChatUtil::fileWrite($logfile, $logEntry, $append);

		} catch (Exception $ex) {
			// Silently fail as before
		}
	}

	public static function getSpiderLogText(){

		$file_path = TWXChatUtil::getUploadsDir('logs').'twx_spider.log';

		global $wp_filesystem;
		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		if(!$wp_filesystem->exists($file_path)) return '';

		return $wp_filesystem->get_contents($file_path);
	}


	public static function clearProgress() {

		// Clear Spider Progress Log
		try{
			$dir = TWXChatUtil::getUploadsDir('spider_results');
			$logFile = $dir.'twxchatspider_progress.txt';
			if(file_exists($logFile)) wp_delete_file($logFile);
		}catch(Exception $ex){
			// Do nothing
		}
	}	


	public static function appendProgress($message, $andAppendToLog = true) {

		try {
			$dir = TWXChatUtil::getUploadsDir('spider_results');
			TWXChatUtil::makeDir($dir);
			$logfile = $dir.'twxchatspider_progress.txt';
			$logEntry = current_time('mysql') . " - " . $message . "\n";
			TWXChatUtil::fileWrite($logfile, $logEntry, true);

			if($andAppendToLog) TWXChatSpider::log("[progress] $message");

		} catch (Exception $ex) {
			// Silently fail as before
		}
	}	

}

