<?php
/**
 * Content Indexer class for managing content indexing
 *
 * This class handles content indexing for various sources including posts,
 * pages, and PDFs, with support for different content types and formats.
 *
 * @package Askany
 * @since 1.0.0
 */

namespace Askany;

use ask\askany\wpnts_nts_fs;

/**
 * Content Indexer class for managing content indexing
 *
 * @package Askany
 * @since 1.0.0
 */
class ContentIndexer {
	/**
	 * Database instance for content management.
	 *
	 * @var Database
	 */
	private $database;
	/**
	 * Batch size for processing content.
	 *
	 * @var int
	 */
	private $batch_size = 50;

	/**
	 * Lazily created crawler, used to render theme-template pages.
	 *
	 * @var ExternalSourcesHandler|null|false
	 */
	private $external_handler = null;

	/**
	 * Seconds spent rendering template pages in this request (time budget).
	 *
	 * @var float
	 */
	private $render_seconds = 0.0;

	/**
	 * Constructor for ContentIndexer.
	 *
	 * Initializes the database instance and sets up hooks.
	 *
	 * @param Database $database Database instance for content management.
	 */
	public function __construct( $database ) {
		$this->database = $database;
	}

	/**
	 * Initialize hooks for content indexing.
	 *
	 * Sets up actions for reindexing content, saving posts, and deleting posts.
	 *
	 * @return void
	 */
	public function init() {
		add_action('askany_reindex_content', array( $this, 'reindex_all_content' ));

		// A different embedding model makes stored vectors incomparable: queue
		// everything for re-embedding when the model in use actually changes.
		foreach ( array( 'askany_embedding_model', 'askany_api_provider' ) as $option ) {
			add_action( 'update_option_' . $option, array( $this, 'maybe_reset_embeddings_for_model_change' ), 20, 0 );
			add_action( 'add_option_' . $option, array( $this, 'maybe_reset_embeddings_for_model_change' ), 20, 0 );
		}
		// Deferred render of a theme-template page (after a save or when a batch ran out of time).
		add_action('askany_render_template_page', array( $this, 'render_template_page' ), 10, 1);

		// Site changes (posts, pages, products, comments, orders, menus) are recorded
		// by IndexQueue and indexed in the background after editing stops. The
		// instant hooks below are only the fallback when the queue is turned off.
		if ( class_exists( __NAMESPACE__ . '\\IndexQueue' ) && IndexQueue::enabled() ) {
			$this->register_admin_item_hooks();
			return;
		}

		add_action('save_post', array( $this, 'index_post' ), 10, 3);
		add_action('delete_post', array( $this, 'delete_post_index' ));
		add_action('wp_insert_comment', array( $this, 'index_comment' ), 10, 2);
		add_action('delete_comment', array( $this, 'delete_comment_index' ));

		// WooCommerce Products & Taxonomies
		add_action('woocommerce_new_product', array( $this, 'index_product' ));
		add_action('woocommerce_update_product', array( $this, 'index_product' ));
		add_action('woocommerce_delete_product', array( $this, 'delete_product_index' ));
		add_action('created_product_cat', array( $this, 'index_product_categories' ));
		add_action('edited_product_cat', array( $this, 'index_product_categories' ));
		add_action('delete_product_cat', array( $this, 'index_product_categories' ));
		add_action('created_product_tag', array( $this, 'index_product_tags' ));
		add_action('edited_product_tag', array( $this, 'index_product_tags' ));
		add_action('delete_product_tag', array( $this, 'index_product_tags' ));
		add_action('woocommerce_attribute_added', array( $this, 'index_product_attributes' ));
		add_action('woocommerce_attribute_updated', array( $this, 'index_product_attributes' ));
		add_action('woocommerce_attribute_deleted', array( $this, 'index_product_attributes' ));

		// WooCommerce Orders Real-Time Hooks
		add_action('woocommerce_new_order', array( $this, 'index_order' ), 10, 1);
		add_action('woocommerce_update_order', array( $this, 'index_order' ), 10, 1);
		add_action('woocommerce_order_status_changed', array( $this, 'index_order' ), 10, 1);
		add_action('woocommerce_trash_order', array( $this, 'delete_order_index' ), 10, 1);
		add_action('woocommerce_delete_order', array( $this, 'delete_order_index' ), 10, 1);

		// Navigation Menus & Custom Links Real-Time Hooks
		add_action('wp_update_nav_menu', array( $this, 'index_menus' ));
		add_action('wp_delete_nav_menu', array( $this, 'index_menus' ));

		$this->register_admin_item_hooks();
	}

	/**
	 * Items the admin adds inside AskAny (Q&A, FAQs, PDFs, External Sources) are
	 * indexed right away — the admin is waiting for them and they are not edited
	 * repeatedly like posts.
	 *
	 * @return void
	 */
	private function register_admin_item_hooks() {
		// Custom Action Hooks for Q&A, FAQs, PDFs, External Sources
		add_action('askany_qa_pair_saved', array( $this, 'index_single_qa_pair' ), 10, 1);
		add_action('askany_qa_pair_deleted', array( $this, 'delete_qa_pair_index' ), 10, 1);
		add_action('askany_faq_saved', array( $this, 'index_single_faq' ), 10, 1);
		add_action('askany_faq_deleted', array( $this, 'delete_faq_index' ), 10, 1);
		add_action('askany_pdf_saved', array( $this, 'index_single_pdf' ), 10, 1);
		add_action('askany_pdf_deleted', array( $this, 'delete_pdf_index' ), 10, 1);
		add_action('askany_external_source_saved', array( $this, 'index_single_external_source' ), 10, 1);
		add_action('askany_external_source_deleted', array( $this, 'delete_external_source_index' ), 10, 1);
	}

	/**
	 * Reindex all content.
	 *
	 * This method is called to reindex all content types including posts, comments,
	 * and products. It clears the existing index and rebuilds it.
	 *
	 * @return bool True on success, false on failure.
	 */
	public function index_content() {
		return $this->reindex_all_content();
	}

	/**
	 * Reindex all content types.
	 *
	 * This method clears the existing index and rebuilds it for all content types.
	 *
	 * @return bool True on success, false on failure.
	 */
	public function reindex_all_content() {
		global $wpdb;
		$tables = $this->database->get_tables();
		$content_index_table = $tables['content_index'];

		// Refresh in place (no TRUNCATE): unchanged items keep their row id and
		// embeddings, so a full re-index does not re-embed everything. Rows not
		// seen in this run are removed at the end.
		$started = current_time( 'mysql' );
		if ( function_exists( 'set_time_limit' ) ) {
			@set_time_limit( 0 ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged
		}

		// Index posts and pages.
		$this->index_posts();
		$this->index_pages();
		$this->index_template_pages();
		$this->index_docs_pages();

		// Index comments.
		$this->index_comments();

		// Index products and related content if WooCommerce is active.
		if ( class_exists('WooCommerce') ) {
			$this->index_products();
			$this->index_product_categories();
			$this->index_product_tags();
			$this->index_product_attributes();
			$this->index_store_configuration();
			$this->index_orders();
		}

		// Index FAQ data for better AI understanding
		$this->index_faqs();
		
		// Index PDF content
		$this->index_pdfs();
		
		// Index external sources
		$this->index_external_sources();

		// Index custom database tables
		$this->index_custom_tables();

		// Index additional website content.
		$this->index_custom_fields();
		$this->index_menus();
		$this->index_widgets();
		$this->index_sitemaps();

		// Remove content that no longer exists (not refreshed in this run) and its vectors.
		$embeddings_table = $wpdb->prefix . 'askany_embeddings';
		// phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared
		$wpdb->query( $wpdb->prepare( "DELETE e FROM {$embeddings_table} e JOIN {$content_index_table} c ON c.id = e.content_id WHERE c.updated_at < %s AND c.object_type <> 'external_url'", $started ) );
		$wpdb->query( $wpdb->prepare( "DELETE FROM {$content_index_table} WHERE updated_at < %s AND object_type <> 'external_url'", $started ) );
		// phpcs:enable
		update_option( 'askany_last_full_reindex', current_time( 'mysql' ), false );
		do_action( 'askany_full_reindex_completed', $started );

		return true;
	}

	/**
	 * When the embedding model in use changes, mark every item for re-embedding.
	 * Search already ignores vectors from other models, so answers fall back to
	 * keyword search (never to mixed, meaningless vectors) until the admin runs
	 * "Generate embeddings" on the RAG page.
	 *
	 * @return void
	 */
	public function maybe_reset_embeddings_for_model_change() {
		global $wpdb;
		$manager = new Embedding_Manager( $this->database, new ApiManager() );
		$model   = $manager->get_active_model();
		if ( '' === $model ) {
			return;
		}

		$previous = (string) get_option( 'askany_embedding_effective_model', '' );
		if ( '' === $previous ) {
			// First run on an existing site: the model most vectors were made with.
			// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
			$previous = (string) $wpdb->get_var( "SELECT embedding_model FROM {$wpdb->prefix}askany_embeddings GROUP BY embedding_model ORDER BY COUNT(*) DESC LIMIT 1" );
		}
		update_option( 'askany_embedding_effective_model', $model, false );

		if ( '' === $previous || $previous === $model ) {
			return;
		}

		$tables = $this->database->get_tables();
		// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
		$wpdb->query( "UPDATE {$tables['content_index']} SET has_embedding = 0, embedding_status = 'pending', embedding_attempts = 0, embedding_error = NULL WHERE object_type <> 'woocommerce_order'" );
		do_action( 'askany_embedding_model_changed', $previous, $model );
	}

	/**
	 * Index or remove one changed item (called by IndexQueue in the background).
	 *
	 * @param string $type   post|page|product|order|comment|product_categories|product_tags|product_attributes|menus.
	 * @param int    $id     Object ID.
	 * @param string $action index|delete.
	 * @return void
	 */
	public function process_queue_item( $type, $id, $action = 'index' ) {
		$id     = (int) $id;
		$delete = ( 'delete' === $action );

		switch ( $type ) {
			case 'page':
				if ( $delete ) {
					$this->delete_post_index( $id );
				} else {
					$this->remove_index_row( 'post', $id ); // old duplicate copy
					$this->index_page( $id, true );
				}
				break;
			case 'post':
				$delete ? $this->delete_post_index( $id ) : $this->index_post( $id );
				break;
			case 'product':
				if ( $delete || 'publish' !== get_post_status( $id ) ) {
					$this->delete_product_index( $id );
				} else {
					$this->index_product( $id );
				}
				break;
			case 'order':
				$delete ? $this->delete_order_index( $id ) : $this->index_order( $id );
				break;
			case 'comment':
				$delete ? $this->delete_comment_index( $id ) : $this->index_comment( $id );
				break;
			case 'product_categories':
				$this->index_product_categories();
				break;
			case 'product_tags':
				$this->index_product_tags();
				break;
			case 'product_attributes':
				$this->index_product_attributes();
				break;
			case 'menus':
				$this->index_menus();
				break;
			default:
				/**
				 * Handle a custom queued item type.
				 *
				 * @param int    $id     Object ID.
				 * @param string $action index|delete.
				 */
				do_action( 'askany_process_queue_item_' . sanitize_key( $type ), $id, $action );
		}
	}

	public function get_indexing_tasks() {
		$wp_totals = $this->get_wordpress_content_totals();
		$source_totals = $this->get_source_content_totals();

		$tasks = array(
			array('type' => 'posts', 'label' => 'Posts', 'total' => $wp_totals['posts'] ?? 0),
			array('type' => 'pages', 'label' => 'Pages', 'total' => $wp_totals['pages'] ?? 0),
			array('type' => 'template_pages', 'label' => 'Template Pages', 'total' => $wp_totals['template_pages'] ?? 0),
			array('type' => 'docs', 'label' => 'Documentation', 'total' => $wp_totals['docs'] ?? 0),
			array('type' => 'comments', 'label' => 'Comments', 'total' => $wp_totals['comments'] ?? 0),
		);

		if ( class_exists('WooCommerce') ) {
			$tasks[] = array('type' => 'products', 'label' => 'Products', 'total' => $wp_totals['products'] ?? 0);
			$tasks[] = array('type' => 'product_categories', 'label' => 'Product Categories', 'total' => $wp_totals['product_categories'] ?? 0);
			$tasks[] = array('type' => 'product_tags', 'label' => 'Product Tags', 'total' => $wp_totals['product_tags'] ?? 0);
			$tasks[] = array('type' => 'product_attributes', 'label' => 'Product Attributes', 'total' => $wp_totals['product_attributes'] ?? 0);
			$tasks[] = array('type' => 'orders', 'label' => 'WooCommerce Orders', 'total' => $wp_totals['orders'] ?? 0);
			$tasks[] = array('type' => 'store_config', 'label' => 'Store Config', 'total' => 1);
		}

		$tasks[] = array('type' => 'faqs', 'label' => 'FAQs', 'total' => $source_totals['faqs'] ?? 0);
		$tasks[] = array('type' => 'qa_pairs', 'label' => 'Q&A Pairs', 'total' => $source_totals['qa_pairs'] ?? 0);
		$tasks[] = array('type' => 'pdfs', 'label' => 'PDFs', 'total' => $source_totals['pdfs'] ?? 0);
		$tasks[] = array('type' => 'external_sources', 'label' => 'External Sources', 'total' => $source_totals['external_sources'] ?? 0);

		$custom_tables = get_option( 'askany_custom_db_tables', array() );
		if ( ! empty( $custom_tables ) && is_array( $custom_tables ) ) {
			$tasks[] = array('type' => 'custom_tables', 'label' => 'Custom DB Tables', 'total' => count($custom_tables));
		}

		$tasks[] = array('type' => 'custom_fields', 'label' => 'Custom Fields', 'total' => $wp_totals['custom_fields'] ?? 0);
		$tasks[] = array('type' => 'menus', 'label' => 'Menus', 'total' => $wp_totals['menus'] ?? 0);
		$tasks[] = array('type' => 'widgets', 'label' => 'Widgets', 'total' => $wp_totals['widgets'] ?? 0);
		$tasks[] = array('type' => 'sitemaps', 'label' => 'Sitemaps', 'total' => 1);

		// Filter out tasks with 0 total (except those that we might not know exact count for)
		$filtered_tasks = array();
		foreach ($tasks as $task) {
			if ($task['total'] > 0 || in_array($task['type'], ['store_config', 'custom_fields', 'widgets', 'menus', 'sitemaps', 'docs', 'orders', 'custom_tables'])) {
				// Set total to at least 1 for progress math if it's 0
				if ($task['total'] == 0) $task['total'] = 1;
				$filtered_tasks[] = $task;
			}
		}

		return $filtered_tasks;
	}

	public function process_task_batch($task_type, $offset, $limit) {
		$processed = 0;
		$done = false;
		$logs = array();

		try {
			switch ($task_type) {
				case 'posts':
					$post_types = get_post_types(array('public' => true));
					unset($post_types['attachment'], $post_types['page']); // Exclude page, handled separately
					$args = array('post_type' => $post_types, 'post_status' => 'publish', 'posts_per_page' => $limit, 'offset' => $offset, 'fields' => 'ids');
					$ids = get_posts($args);
					foreach($ids as $id) { $this->index_post($id); $processed++; }
					$done = (count($ids) < $limit);
					$logs[] = array('type' => 'info', 'message' => "Indexed $processed posts in this batch.");
					break;
				case 'pages':
					$args = array('post_type' => 'page', 'post_status' => 'publish', 'posts_per_page' => $limit, 'offset' => $offset, 'fields' => 'ids');
					$ids = get_posts($args);
					foreach($ids as $id) { $this->index_page($id); $processed++; }
					$done = (count($ids) < $limit);
					$logs[] = array('type' => 'info', 'message' => "Indexed $processed pages in this batch.");
					break;
				case 'comments':
					$args = array('status' => 'approve', 'number' => $limit, 'offset' => $offset, 'fields' => 'ids');
					$ids = get_comments($args);
					foreach($ids as $id) { $this->index_comment($id); $processed++; }
					$done = (count($ids) < $limit);
					$logs[] = array('type' => 'info', 'message' => "Indexed $processed comments in this batch.");
					break;
				case 'products':
					$args = array('post_type' => 'product', 'post_status' => 'publish', 'posts_per_page' => $limit, 'offset' => $offset, 'fields' => 'ids');
					$ids = get_posts($args);
					foreach($ids as $id) { $this->index_product($id); $processed++; }
					$done = (count($ids) < $limit);
					$logs[] = array('type' => 'info', 'message' => "Indexed $processed products in this batch.");
					break;
				case 'orders':
					if ( function_exists('wc_get_orders') ) {
						// Same cap as the full re-index: the newest N orders only.
						$cap  = max( 1, (int) apply_filters( 'askany_index_orders_limit', 200 ) );
						$take = max( 0, min( (int) $limit, $cap - (int) $offset ) );
						$order_ids = $take > 0 ? wc_get_orders( array('limit' => $take, 'offset' => $offset, 'orderby' => 'date', 'order' => 'DESC', 'return' => 'ids') ) : array();
						foreach ( $order_ids as $id ) { $this->index_order($id); $processed++; }
						$done = ( count($order_ids) < $take ) || ( (int) $offset + count( $order_ids ) >= $cap );
					} else {
						$done = true;
					}
					$logs[] = array('type' => 'info', 'message' => "Indexed $processed orders in this batch.");
					break;
				case 'product_attributes':
					if ($offset == 0) {
						$this->index_product_attributes();
						$processed = 1;
					}
					$done = true;
					$logs[] = array('type' => 'info', 'message' => "Completed indexing product_attributes.");
					break;
				case 'custom_tables':
					if ($offset == 0) {
						$this->index_custom_tables();
						$processed = 1;
					}
					$done = true;
					$logs[] = array('type' => 'info', 'message' => "Completed indexing custom database tables.");
					break;
				default:
					if ($offset == 0) {
						$method = 'index_' . $task_type;
						// Handle specific method mappings
						if ($task_type === 'faqs') $method = 'index_home_faqs';
						if ($task_type === 'docs') $method = 'index_docs_pages';
						if ($task_type === 'sitemaps') $method = 'index_sitemaps';
						if ($task_type === 'orders') $method = 'index_orders';
						if (method_exists($this, $method)) {
							$this->$method();
							$processed = 1; 
						}
						$done = true;
						$logs[] = array('type' => 'info', 'message' => "Completed indexing $task_type.");
					} else {
						$done = true;
					}
					break;
			}
		} catch (\Exception $e) {
			$logs[] = array('type' => 'error', 'message' => "Error indexing $task_type: " . $e->getMessage());
			$done = true;
		}

		return array(
			'processed' => $processed,
			'done' => $done,
			'logs' => $logs
		);
	}

	/**
	 * Get comprehensive content statistics.
	 *
	 * This method retrieves detailed statistics about the indexed content including
	 * counts for all content types, total content size, and indexing status.
	 *
	 * @return array Array containing detailed content statistics.
	 */
	public function get_content_stats() {
		global $wpdb;
		$tables = $this->database->get_tables();
		$content_index_table = $tables['content_index'];

		// Get individual content type counts
		$stats = array(
			'posts' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'post')),
			'pages' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'page')),
			'template_pages' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'template_page')),
			'docs' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'docs')),
			'comments' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'comment')),
			'products' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'product')),
			'product_categories' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'product_category')),
			'product_tags' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'product_tag')),
			'product_attributes' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'product_attribute')),
			'orders' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'woocommerce_order')),
			'custom_fields' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'custom_field')),
			'menus' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'menu')),
			'widgets' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'widget')),
			'store_config' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'store_config')),
			// FAQ, Q&A, PDF, and External Sources
			'faqs' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'faq')),
			'faq_collections' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'faq_collection')),
			'qa_pairs' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'qa')),
			'qa_collections' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'qa_collection')),
			'pdfs' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'pdf')),
			'pdf_collections' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'pdf_collection')),
			'external_sources' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'external_url')),
			'external_collections' => (int) $wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $content_index_table WHERE object_type = %s", 'external_collection')),
		);

		// Add custom database table stats
		$custom_table_stats            = $this->database->get_custom_table_stats();
		$stats['custom_tables']         = $custom_table_stats['table_count'];
		$stats['custom_tables_records'] = $custom_table_stats['estimated_records'];

		// Conditionally zero out WooCommerce stats if toggled off
		if ( ! get_option('askany_woocommerce_enabled', true) ) {
			$stats['products'] = 0;
			$stats['product_categories'] = 0;
			$stats['product_tags'] = 0;
			$stats['product_attributes'] = 0;
			$stats['orders'] = 0;
			$stats['store_config'] = 0;
		}

		// Calculate totals and additional metrics
		$stats['total_indexed'] = array_sum($stats);
		
		// Get content size statistics
		$stats['total_content_size'] = (int) $wpdb->get_var("SELECT SUM(LENGTH(content)) FROM $content_index_table");
		$stats['average_content_length'] = $stats['total_indexed'] > 0 ? 
			round($stats['total_content_size'] / $stats['total_indexed']) : 0;

		// Get last update information
		$stats['last_updated'] = $wpdb->get_var("SELECT MAX(updated_at) FROM $content_index_table");
		
		// Get content type breakdown with percentages
		$stats['content_breakdown'] = array();
		$content_types = array('posts', 'pages', 'template_pages', 'docs', 'comments', 'products', 'product_categories', 'product_tags', 'product_attributes', 'orders', 'custom_fields', 'menus', 'widgets', 'store_config', 'faqs', 'faq_collections', 'qa_pairs', 'qa_collections', 'pdfs', 'pdf_collections', 'external_sources', 'external_collections');
		
		foreach ( $content_types as $type ) {
			$stats['content_breakdown'][$type] = array(
				'count' => $stats[$type],
				'percentage' => $stats['total_indexed'] > 0 ? 
					round(($stats[$type] / $stats['total_indexed']) * 100, 2) : 0,
			);
		}

		// Get WordPress content availability for comparison
		$stats['wordpress_totals'] = $this->get_wordpress_content_totals();
		
		// Get FAQ, PDF, and External Sources totals from their respective tables
		$stats['source_totals'] = $this->get_source_content_totals();
		
		// Calculate indexing coverage
		$stats['indexing_coverage'] = array();
		if ( $stats['wordpress_totals']['posts'] > 0 ) {
			$stats['indexing_coverage']['posts'] = round(($stats['posts'] / $stats['wordpress_totals']['posts']) * 100, 2);
		}
		if ( $stats['wordpress_totals']['pages'] > 0 ) {
			$stats['indexing_coverage']['pages'] = round(($stats['pages'] / $stats['wordpress_totals']['pages']) * 100, 2);
		}
		if ( $stats['wordpress_totals']['comments'] > 0 ) {
			$stats['indexing_coverage']['comments'] = round(($stats['comments'] / $stats['wordpress_totals']['comments']) * 100, 2);
		}
		if ( class_exists('WooCommerce') && $stats['wordpress_totals']['products'] > 0 ) {
			$stats['indexing_coverage']['products'] = round(($stats['products'] / $stats['wordpress_totals']['products']) * 100, 2);
		}
		
		// Calculate coverage for new content types
		if ( $stats['source_totals']['faqs'] > 0 ) {
			$stats['indexing_coverage']['faqs'] = round(($stats['faqs'] / $stats['source_totals']['faqs']) * 100, 2);
		}
		if ( $stats['source_totals']['qa_pairs'] > 0 ) {
			$stats['indexing_coverage']['qa_pairs'] = round(($stats['qa_pairs'] / $stats['source_totals']['qa_pairs']) * 100, 2);
		}
		if ( $stats['source_totals']['pdfs'] > 0 ) {
			$stats['indexing_coverage']['pdfs'] = round(($stats['pdfs'] / $stats['source_totals']['pdfs']) * 100, 2);
		}
		if ( $stats['source_totals']['external_sources'] > 0 ) {
			$stats['indexing_coverage']['external_sources'] = round(($stats['external_sources'] / $stats['source_totals']['external_sources']) * 100, 2);
		}

		// Add indexing health status
		$stats['indexing_health'] = $this->calculate_indexing_health($stats);

		return $stats;
	}

	/**
	 * Get WordPress content totals for comparison.
	 *
	 * @return array Array containing WordPress content totals.
	 */
	private function get_wordpress_content_totals() {
		$totals = array(
			'posts' => (int) wp_count_posts('post')->publish,
			'pages' => (int) wp_count_posts('page')->publish,
			'comments' => (int) wp_count_comments()->approved,
		);

		// Add WooCommerce product count if available
		if ( class_exists('WooCommerce') ) {
			$totals['products'] = (int) wp_count_posts('product')->publish;
			
			// Get product categories and tags count
			$totals['product_categories'] = (int) wp_count_terms(array(
				'taxonomy' => 'product_cat',
				'hide_empty' => false,
			));
			
			$totals['product_tags'] = (int) wp_count_terms(array(
				'taxonomy' => 'product_tag',
				'hide_empty' => false,
			));
			
			// Get product attributes count
			$totals['product_attributes'] = 0;
			$attribute_taxonomies = wc_get_attribute_taxonomies();
			if ( ! empty( $attribute_taxonomies ) ) {
				$tax_names = array();
				foreach ( $attribute_taxonomies as $tax ) {
					$tax_names[] = wc_attribute_taxonomy_name( $tax->attribute_name );
				}
				$totals['product_attributes'] = (int) wp_count_terms(array(
					'taxonomy' => $tax_names,
					'hide_empty' => false,
				));
			}

			if ( function_exists( 'wc_get_orders' ) ) {
				$totals['orders'] = count( wc_get_orders( array( 'limit' => -1, 'return' => 'ids' ) ) );
			} else {
				$totals['orders'] = (int) wp_count_posts( 'shop_order' )->publish;
			}
		}

		// Get menu count
		$totals['menus'] = count(wp_get_nav_menus());

		// Get template count
		$totals['template_pages'] = count(wp_get_theme()->get_page_templates());

		// Get documentation post types count (including BetterDocs & all CPTs)
		$doc_post_types = $this->get_all_doc_and_cpt_types();
		$totals['docs'] = 0;
		foreach ( $doc_post_types as $post_type ) {
			if ( post_type_exists($post_type) ) {
				$count = wp_count_posts($post_type);
				$totals['docs'] += isset($count->publish) ? (int) $count->publish : 0;
			}
		}

		// Estimate custom fields count (this is approximate)
		global $wpdb;
		$totals['custom_fields'] = (int) $wpdb->get_var("
			SELECT COUNT(DISTINCT CONCAT(post_id, '_', meta_key)) 
			FROM {$wpdb->postmeta} 
			WHERE meta_key NOT LIKE '\_%' 
			AND meta_value != '' 
			AND LENGTH(meta_value) > 10
		");

		// Estimate widgets count
		$sidebars = wp_get_sidebars_widgets();
		$widget_count = 0;
		foreach ( $sidebars as $sidebar_id => $widgets ) {
			// Inactive / orphaned widgets are not shown on the site.
			if ( 'wp_inactive_widgets' === $sidebar_id || 0 === strpos( (string) $sidebar_id, 'orphaned_widgets' ) ) {
				continue;
			}
			if ( is_array($widgets) ) {
				$widget_count += count($widgets);
			}
		}
		$totals['widgets'] = $widget_count;

		return $totals;
	}
	
	/**
	 * Get source content totals (FAQs, PDFs, External Sources) for comparison.
	 *
	 * This method retrieves the total counts of FAQs, PDFs, and External Sources
	 * from their respective database tables for coverage calculation.
	 *
	 * @return array Array containing source content totals.
	 */
	private function get_source_content_totals() {
		global $wpdb;
		$tables = $this->database->get_tables();
		
		$totals = array(
			'faqs' => 0,
			'qa_pairs' => 0,
			'pdfs' => 0,
			'external_sources' => 0,
		);
		
		// Get FAQ totals (home FAQs)
		$faqs_table = $tables['home_faqs'];
		if ( $wpdb->get_var("SHOW TABLES LIKE '$faqs_table'") === $faqs_table ) {
			$totals['faqs'] = (int) $wpdb->get_var("SELECT COUNT(*) FROM $faqs_table");
		}
		
		// Get Q&A pairs totals
		$qa_table = $tables['qa_pairs'];
		if ( $wpdb->get_var("SHOW TABLES LIKE '$qa_table'") === $qa_table ) {
			$totals['qa_pairs'] = (int) $wpdb->get_var("SELECT COUNT(*) FROM $qa_table WHERE status = 'active'");
		}
		
		// Get PDF totals
		$pdf_table = $tables['pdf_index'];
		if ( $wpdb->get_var("SHOW TABLES LIKE '$pdf_table'") === $pdf_table ) {
			$totals['pdfs'] = (int) $wpdb->get_var("SELECT COUNT(*) FROM $pdf_table");
		}
		
		// Get External Sources totals
		$external_table = $tables['external_sources'];
		if ( $wpdb->get_var("SHOW TABLES LIKE '$external_table'") === $external_table ) {
			$totals['external_sources'] = (int) $wpdb->get_var("SELECT COUNT(*) FROM $external_table");
		}
		
		return $totals;
	}

	/**
	 * Calculate indexing health status.
	 *
	 * @param array $stats Current statistics.
	 * @return array Health status information.
	 */
	private function calculate_indexing_health( $stats ) {
		$health = array(
			'status' => 'good',
			'score' => 100,
			'issues' => array(),
			'recommendations' => array(),
		);

		// Check if there's any content indexed
		if ( $stats['total_indexed'] === 0 ) {
			$health['status'] = 'critical';
			$health['score'] = 0;
			$health['issues'][] = 'No content has been indexed yet';
			$health['recommendations'][] = 'Run a full content reindex to populate the search index';
			return $health;
		}

		// Check coverage for main content types
		$coverage_issues = 0;
		if ( isset($stats['indexing_coverage']['posts']) && $stats['indexing_coverage']['posts'] < 90 ) {
			$coverage_issues++;
			$health['issues'][] = 'Post indexing coverage is below 90%';
		}
		
		if ( isset($stats['indexing_coverage']['pages']) && $stats['indexing_coverage']['pages'] < 90 ) {
			$coverage_issues++;
			$health['issues'][] = 'Page indexing coverage is below 90%';
		}

		if ( isset($stats['indexing_coverage']['products']) && $stats['indexing_coverage']['products'] < 90 ) {
			$coverage_issues++;
			$health['issues'][] = 'Product indexing coverage is below 90%';
		}

		// Check if index is stale (older than 7 days)
		if ( $stats['last_updated'] ) {
			$last_update = strtotime($stats['last_updated']);
			$days_old = (time() - $last_update) / (24 * 60 * 60);
			
			if ( $days_old > 7 ) {
				$coverage_issues++;
				$health['issues'][] = 'Content index is more than 7 days old';
				$health['recommendations'][] = 'Consider running a content reindex to ensure freshness';
			}
		}

		// Calculate health score based on issues
		if ( $coverage_issues > 0 ) {
			$health['score'] = max(0, 100 - ($coverage_issues * 20));
			
			if ( $health['score'] < 60 ) {
				$health['status'] = 'critical';
			} elseif ( $health['score'] < 80 ) {
				$health['status'] = 'warning';
			}
		}

		// Add general recommendations
		if ( $stats['total_indexed'] < 100 ) {
			$health['recommendations'][] = 'Consider adding more content to improve AI response quality';
		}

		if ( empty($health['recommendations']) ) {
			$health['recommendations'][] = 'Content indexing is working well. Monitor regularly for optimal performance.';
		}

		return $health;
	}

	/**
	 * Index posts in batches.
	 *
	 * This method retrieves all posts and indexes them in batches to improve performance.
	 *
	 * @return void
	 */
	private function index_posts() {
		$post_types = get_post_types(array( 'public' => true ));
		// Pages are indexed by index_pages() (and products by index_products()).
		unset($post_types['attachment'], $post_types['page'], $post_types['product']);

		$args = array(
			'post_type' => $post_types,
			'post_status' => 'publish',
			'posts_per_page' => -1,
			'fields' => 'ids',
		);

		$post_ids = get_posts($args);
		$total_posts = count($post_ids);

		for ( $i = 0; $i < $total_posts; $i += $this->batch_size ) {
			$batch = array_slice($post_ids, $i, $this->batch_size);
			foreach ( $batch as $post_id ) {
				$this->index_post($post_id);
			}
		}
	}

	/**
	 * Index comments in batches.
	 *
	 * This method retrieves all approved comments and indexes them in batches to improve performance.
	 *
	 * @return void
	 */
	private function index_comments() {
		$args = array(
			'status' => 'approve',
			'number' => -1,
			'fields' => 'ids',
		);

		$comment_ids = get_comments($args);
		$total_comments = count($comment_ids);

		for ( $i = 0; $i < $total_comments; $i += $this->batch_size ) {
			$batch = array_slice($comment_ids, $i, $this->batch_size);
			foreach ( $batch as $comment_id ) {
				$this->index_comment($comment_id);
			}
		}
	}

	/**
	 * Index products in batches.
	 *
	 * This method retrieves all published products and indexes them in batches to improve performance.
	 *
	 * @return void
	 */
	private function index_products() {
		$args = array(
			'post_type' => 'product',
			'post_status' => 'publish',
			'posts_per_page' => -1,
			'fields' => 'ids',
		);

		$product_ids = get_posts($args);
		$total_products = count($product_ids);

		for ( $i = 0; $i < $total_products; $i += $this->batch_size ) {
			$batch = array_slice($product_ids, $i, $this->batch_size);
			foreach ( $batch as $product_id ) {
				$this->index_product($product_id);
			}
		}
	}

	/**
	 * Index a single post.
	 *
	 * This method indexes a single post by its ID. It prepares the content and stores it in the database.
	 *
	 * @param int          $post_id Post ID to index.
	 * @param WP_Post|null $post Optional. Post object. Default null.
	 * @param bool|null    $update Optional. Whether this is an update. Default null.
	 * @return void
	 */
	public function index_post( $post_id, $post = null, $update = null ) {
		if ( defined('DOING_AUTOSAVE') && DOING_AUTOSAVE ) {
			return;
		}

		if ( ! $post ) {
			$post = get_post($post_id);
		}

		if ( ! $post || $post->post_status !== 'publish' ) {
			// A page taken offline (draft/private/trash) must leave the knowledge base.
			if ( $post && 'page' === $post->post_type ) {
				$this->delete_post_index( $post_id );
			}
			return;
		}

		// Products have their own, richer index entry (index_product); menus, styles,
		// templates and other non-public types are not knowledge; protected posts are private.
		if ( in_array( $post->post_type, array( 'product', 'product_variation', 'shop_order', 'shop_coupon', 'attachment' ), true )
			|| ! is_post_type_viewable( $post->post_type )
			|| ! empty( $post->post_password ) ) {
			return;
		}

		// Pages have their own index row ('page'); storing them here too as 'post'
		// created a duplicate copy of every saved page until the next full re-index.
		if ( 'page' === $post->post_type ) {
			$this->remove_index_row( 'post', $post_id );
			if ( $this->get_classic_template( $post_id ) ) {
				// Render after the save finishes (page caches purge first); keep the
				// current row meanwhile so the knowledge is never empty.
				if ( ! wp_next_scheduled( 'askany_render_template_page', array( (int) $post_id ) ) ) {
					wp_schedule_single_event( time() + 20, 'askany_render_template_page', array( (int) $post_id ) );
				}
			} else {
				$this->index_page( $post_id );
			}
			return;
		}

		// Store the post object before modifying global
		$current_post = $post;
		
		// Set global post for Elementor extraction
		global $post;
		$old_global_post = $post;
		$post = get_post($post_id);
		
		// Verify post was retrieved successfully
		if ( ! $post ) {
			$post = $old_global_post;
			return;
		}
		
		// Combine title, content, and excerpt
		$content_parts = array(
			'Title: ' . $post->post_title,
			'Content: ' . $post->post_content,
		);
		
		if ( ! empty( $post->post_excerpt ) ) {
			$content_parts[] = 'Excerpt: ' . $post->post_excerpt;
		}
		
		// Add post metadata for better AI context
		$author = get_the_author_meta('display_name', $post->post_author);
		if ( $author ) {
			$content_parts[] = 'Author: ' . $author;
		}
		
		$content_parts[] = 'Published: ' . get_the_date('F j, Y', $post_id);
		$content_parts[] = 'Modified: ' . get_the_modified_date('F j, Y', $post_id);
		$content_parts[] = 'Post Type: ' . $post->post_type;
		
		// Add categories with descriptions
		$categories = get_the_category($post_id);
		if ( ! empty($categories) ) {
			$cat_info = array();
			foreach ( $categories as $category ) {
				$cat_text = 'Category: ' . $category->name;
				if ( ! empty($category->description) ) {
					$cat_text .= ' - ' . $category->description;
				}
				$cat_info[] = $cat_text;
			}
			$content_parts = array_merge($content_parts, $cat_info);
		}
		
		// Add tags
		$tags = get_the_tags($post_id);
		if ( ! empty($tags) ) {
			$tag_names = array_map(function($tag) { return $tag->name; }, $tags);
			$content_parts[] = 'Tags: ' . implode(', ', $tag_names);
		}
		
		// Add custom taxonomies
		$taxonomies = get_object_taxonomies($post->post_type, 'objects');
		foreach ( $taxonomies as $taxonomy ) {
			// Skip built-in taxonomies already handled
			if ( in_array($taxonomy->name, array('category', 'post_tag', 'post_format')) ) {
				continue;
			}
			
			$terms = get_the_terms($post_id, $taxonomy->name);
			if ( ! empty($terms) && ! is_wp_error($terms) ) {
				foreach ( $terms as $term ) {
					$term_text = $taxonomy->label . ': ' . $term->name;
					if ( ! empty($term->description) ) {
						$term_text .= ' - ' . $term->description;
					}
					$content_parts[] = $term_text;
				}
			}
		}
		
		// Add featured image alt text
		if ( has_post_thumbnail($post_id) ) {
			$thumbnail_id = get_post_thumbnail_id($post_id);
			$alt_text = get_post_meta($thumbnail_id, '_wp_attachment_image_alt', true);
			if ( $alt_text ) {
				$content_parts[] = 'Featured Image: ' . $alt_text;
			}
			$caption = wp_get_attachment_caption($thumbnail_id);
			if ( $caption ) {
				$content_parts[] = 'Image Caption: ' . $caption;
			}
		}
		
		// Add ACF fields if ACF is active
		if ( function_exists('get_fields') ) {
			$acf_fields = get_fields($post_id);
			if ( ! empty($acf_fields) && is_array($acf_fields) ) {
				foreach ( $acf_fields as $field_name => $field_value ) {
					// Skip empty values and arrays/objects
					if ( empty($field_value) || is_array($field_value) || is_object($field_value) ) {
						continue;
					}
					// Convert field name to readable format
					$readable_name = ucwords(str_replace(array('_', '-'), ' ', $field_name));
					$content_parts[] = $readable_name . ': ' . $field_value;
				}
			}
		}
		
		// Add common custom fields (non-private meta)
		$all_meta = get_post_meta($post_id);
		$allowed_meta_keys = array(
			'subtitle' => 'Subtitle',
			'subheading' => 'Subheading',
			'custom_description' => 'Description',
			'location' => 'Location',
			'event_date' => 'Event Date',
			'price' => 'Price',
			'duration' => 'Duration',
		);
		
		foreach ( $allowed_meta_keys as $meta_key => $label ) {
			if ( isset($all_meta[$meta_key]) && ! empty($all_meta[$meta_key][0]) ) {
				$meta_value = $all_meta[$meta_key][0];
				// Skip serialized data
				if ( ! is_serialized($meta_value) ) {
					$content_parts[] = $label . ': ' . $meta_value;
				}
			}
		}
		
		// Add Elementor meta description if available
		$elementor_settings = get_post_meta( $post_id, '_elementor_page_settings', true );
		if ( ! empty( $elementor_settings['post_excerpt'] ) ) {
			$content_parts[] = 'Meta Description: ' . $elementor_settings['post_excerpt'];
		}
		
		// Add SEO meta if available
		$seo_desc = get_post_meta( $post_id, '_yoast_wpseo_metadesc', true );
		if ( $seo_desc ) {
			$content_parts[] = 'SEO Description: ' . $seo_desc;
		}
		
		$seo_title = get_post_meta( $post_id, '_yoast_wpseo_title', true );
		if ( $seo_title ) {
			$content_parts[] = 'SEO Title: ' . $seo_title;
		}

		
		// Elementor text is added once by prepare_content() (global $post is set).

		$content = $this->prepare_content(implode("\n", $content_parts));
		
		// Store title before restoring global post
		$post_title = $post->post_title;
		
		// Restore global post
		$post = $old_global_post;
		
		if ( empty($content) ) {
			return;
		}

		$this->store_content('post', $post_id, $content, get_permalink($post_id), $post_title);
	}

	/**
	 * Index a single comment.
	 *
	 * This method indexes a single comment by its ID. It prepares the content and stores it in the database.
	 *
	 * @param int             $comment_id Comment ID to index.
	 * @param WP_Comment|null $comment Optional. Comment object. Default null.
	 * @return void
	 */
	public function index_comment( $comment_id, $comment = null ) {
		if ( ! $comment ) {
			$comment = get_comment($comment_id);
		}

		if ( ! $comment || $comment->comment_approved !== '1' ) {
			return;
		}

		// Order notes and system comments contain private data.
		if ( ! in_array( (string) $comment->comment_type, array( '', 'comment', 'review' ), true ) ) {
			return;
		}

		$content = $this->prepare_content($comment->comment_content);
		if ( empty($content) ) {
			return;
		}

		$title = 'Comment by ' . $comment->comment_author;
		$this->store_content('comment', $comment_id, $content, get_comment_link($comment_id), $title);
	}

	/**
	 * Index a single product with comprehensive WooCommerce data.
	 *
	 * This method indexes a single product by its ID with full product data including
	 * categories, tags, attributes, variations, stock info, and metadata.
	 * Optimized for AI understanding with clear labels and structure.
	 *
	 * @param int $product_id Product ID to index.
	 * @return void
	 */
	public function index_product( $product_id ) {
		$product = wc_get_product($product_id);
		if ( ! $product ) {
			return;
		}

		$content = array();
		
		// === BASIC PRODUCT INFORMATION ===
		$content[] = '=== PRODUCT INFORMATION ===';
		$content[] = 'Product Name: ' . $product->get_name();
		$content[] = 'Product Type: ' . ucfirst($product->get_type());
		if ( '' !== (string) $product->get_sku() ) {
			$content[] = 'SKU: ' . $product->get_sku();
		}
		$content[] = 'Status: ' . ucfirst($product->get_status());

		// === PRICING ===
		$content[] = "\n=== PRICING ===";
		$content[] = 'Regular Price: ' . wc_price($product->get_regular_price());
		if ( $product->is_on_sale() ) {
			$content[] = 'Sale Price: ' . wc_price($product->get_sale_price());
			$content[] = 'On Sale: Yes';
		}
		// Scheduled sale window, so "until when is it on sale?" can be answered
		// (checked against the TODAY line in the prompt).
		$sale_from = $product->get_date_on_sale_from();
		$sale_to   = $product->get_date_on_sale_to();
		if ( $sale_from || $sale_to ) {
			$content[] = 'Sale Period: ' . ( $sale_from ? 'from ' . $sale_from->date_i18n( get_option( 'date_format' ) ) . ' ' : '' ) . ( $sale_to ? 'until ' . $sale_to->date_i18n( get_option( 'date_format' ) ) : '' );
		}
		$content[] = 'Display Price: ' . $product->get_price_html();
		
		// === DESCRIPTIONS ===
		$content[] = "\n=== DESCRIPTIONS ===";
		if ( $product->get_short_description() ) {
			$content[] = 'Short Description: ' . $product->get_short_description();
		}
		if ( $product->get_description() ) {
			$content[] = 'Full Description: ' . $product->get_description();
		}

		// === STOCK INFORMATION ===
		$content[] = "\n=== STOCK & AVAILABILITY ===";
		if ( $product->managing_stock() ) {
			// No exact quantity here: it changes with every sale (forcing a new
			// embedding each time) and the chat gets live stock from WooCommerce.
			$content[] = 'Stock Status: ' . ucfirst($product->get_stock_status());
			$content[] = 'Backorders: ' . ucfirst($product->get_backorders());
		} else {
			$content[] = 'Stock Status: ' . ucfirst($product->get_stock_status());
		}
		$content[] = 'In Stock: ' . ($product->is_in_stock() ? 'Yes' : 'No');

		// === CATEGORIES ===
		$categories = wp_get_post_terms($product_id, 'product_cat');
		if ( ! empty($categories) ) {
			$content[] = "\n=== CATEGORIES ===";
			foreach ( $categories as $category ) {
				$cat_text = 'Category: ' . $category->name;
				if ( ! empty($category->description) ) {
					$cat_text .= ' - ' . $category->description;
				}
				$content[] = $cat_text;
			}
		}

		// === TAGS ===
		$tags = wp_get_post_terms($product_id, 'product_tag');
		if ( ! empty($tags) ) {
			$content[] = "\n=== TAGS ===";
			$tag_names = array_map(function($tag) { return $tag->name; }, $tags);
			$content[] = 'Tags: ' . implode(', ', $tag_names);
		}

		// === ATTRIBUTES ===
		$attributes = $product->get_attributes();
		if ( ! empty($attributes) ) {
			$content[] = "\n=== PRODUCT ATTRIBUTES ===";
			foreach ( $attributes as $attribute ) {
				$attribute_name = $attribute->get_name();
				$attribute_label = wc_attribute_label($attribute_name);
				
				if ( $attribute->is_taxonomy() ) {
					$terms = wp_get_post_terms($product_id, $attribute_name);
					$term_names = array_map(function($term) { return $term->name; }, $terms);
					if ( ! empty($term_names) ) {
						$content[] = $attribute_label . ': ' . implode(', ', $term_names);
					}
				} else {
					$options = $attribute->get_options();
					if ( is_array($options) && ! empty($options) ) {
						$content[] = $attribute_label . ': ' . implode(', ', $options);
					}
				}
			}
		}

		// === VARIATIONS (for variable products) ===
		if ( $product->is_type('variable') ) {
			$variations = $product->get_children();
			if ( ! empty($variations) ) {
				$content[] = "\n=== PRODUCT VARIATIONS ===";
				$content[] = 'Total Variations: ' . count($variations);
				
				foreach ( $variations as $variation_id ) {
					$variation = wc_get_product($variation_id);
					if ( $variation ) {
						$content[] = "\n--- Variation ---";
						$content[] = 'Variation Name: ' . $variation->get_name();
						
						if ( $variation->get_sku() ) {
							$content[] = 'Variation SKU: ' . $variation->get_sku();
						}
						
						$content[] = 'Variation Price: ' . $variation->get_price_html();
						
						if ( $variation->managing_stock() ) {
							$content[] = 'Variation Stock: ' . $variation->get_stock_quantity();
						}
						$content[] = 'Variation In Stock: ' . ($variation->is_in_stock() ? 'Yes' : 'No');
						
						// Add variation attributes
						$variation_attributes = $variation->get_variation_attributes();
						if ( ! empty($variation_attributes) ) {
							foreach ( $variation_attributes as $attr_name => $attr_value ) {
								$clean_name = str_replace('attribute_', '', $attr_name);
								$clean_name = wc_attribute_label($clean_name);
								$content[] = $clean_name . ': ' . $attr_value;
							}
						}
					}
				}
			}
		}

		// === DIMENSIONS & SHIPPING ===
		$has_dimensions = false;
		$dimensions_content = array();
		
		if ( $product->get_weight() ) {
			$dimensions_content[] = 'Weight: ' . $product->get_weight() . ' ' . get_option('woocommerce_weight_unit');
			$has_dimensions = true;
		}
		if ( $product->get_length() ) {
			$dimensions_content[] = 'Length: ' . $product->get_length() . ' ' . get_option('woocommerce_dimension_unit');
			$has_dimensions = true;
		}
		if ( $product->get_width() ) {
			$dimensions_content[] = 'Width: ' . $product->get_width() . ' ' . get_option('woocommerce_dimension_unit');
			$has_dimensions = true;
		}
		if ( $product->get_height() ) {
			$dimensions_content[] = 'Height: ' . $product->get_height() . ' ' . get_option('woocommerce_dimension_unit');
			$has_dimensions = true;
		}
		
		if ( $has_dimensions ) {
			$content[] = "\n=== DIMENSIONS & SHIPPING ===";
			$content = array_merge($content, $dimensions_content);
		}
		
		$shipping_class = $product->get_shipping_class();
		if ( $shipping_class ) {
			if ( ! $has_dimensions ) {
				$content[] = "\n=== SHIPPING ===";
			}
			$content[] = 'Shipping Class: ' . $shipping_class;
		}

		// === REVIEWS & RATINGS ===
		$average_rating = $product->get_average_rating();
		$review_count = $product->get_review_count();
		if ( $average_rating > 0 || $review_count > 0 ) {
			$content[] = "\n=== REVIEWS & RATINGS ===";
			$content[] = 'Average Rating: ' . $average_rating . ' out of 5';
			$content[] = 'Total Reviews: ' . $review_count;
		}

		// === TAX INFORMATION ===
		$content[] = "\n=== TAX INFORMATION ===";
		$content[] = 'Tax Status: ' . ucfirst($product->get_tax_status());
		$tax_class = $product->get_tax_class();
		if ( $tax_class ) {
			$content[] = 'Tax Class: ' . $tax_class;
		}

		// === ADDITIONAL INFORMATION ===
		$additional_info = array();
		
		$purchase_note = get_post_meta($product_id, '_purchase_note', true);
		if ( $purchase_note ) {
			$additional_info[] = 'Purchase Note: ' . $purchase_note;
		}
		
		if ( $product->is_type('external') ) {
			$external_url = $product->get_product_url();
			$button_text = $product->get_button_text();
			if ( $external_url ) {
				$additional_info[] = 'External URL: ' . $external_url;
			}
			if ( $button_text ) {
				$additional_info[] = 'Button Text: ' . $button_text;
			}
		}
		
		if ( ! empty($additional_info) ) {
			$content[] = "\n=== ADDITIONAL INFORMATION ===";
			$content = array_merge($content, $additional_info);
		}

		// === IMAGES ===
		$gallery_ids = $product->get_gallery_image_ids();
		if ( ! empty($gallery_ids) ) {
			$content[] = "\n=== PRODUCT IMAGES ===";
			foreach ( $gallery_ids as $image_id ) {
				$alt_text = get_post_meta($image_id, '_wp_attachment_image_alt', true);
				if ( $alt_text ) {
					$content[] = 'Image Description: ' . $alt_text;
				}
			}
		}
		
		// Add featured image alt text
		if ( has_post_thumbnail($product_id) ) {
			$thumbnail_id = get_post_thumbnail_id($product_id);
			$alt_text = get_post_meta($thumbnail_id, '_wp_attachment_image_alt', true);
			if ( $alt_text ) {
				if ( empty($gallery_ids) ) {
					$content[] = "\n=== PRODUCT IMAGES ===";
				}
				$content[] = 'Featured Image: ' . $alt_text;
			}
		}

		// === ACF FIELDS (if available) ===
		if ( function_exists('get_fields') ) {
			$acf_fields = get_fields($product_id);
			if ( ! empty($acf_fields) && is_array($acf_fields) ) {
				$content[] = "\n=== CUSTOM FIELDS ===";
				foreach ( $acf_fields as $field_name => $field_value ) {
					if ( empty($field_value) || is_array($field_value) || is_object($field_value) ) {
						continue;
					}
					$readable_name = ucwords(str_replace(array('_', '-'), ' ', $field_name));
					$content[] = $readable_name . ': ' . $field_value;
				}
			}
		}

		$prepared_content = $this->prepare_content(implode("\n", $content));
		if ( empty($prepared_content) ) {
			return;
		}

		$this->store_content('product', $product_id, $prepared_content, get_permalink($product_id), $product->get_name());
	}

	/**
	 * Delete the index for a post.
	 *
	 * This method deletes the index entry for a specific post by its ID.
	 *
	 * @param int $post_id Post ID to delete from the index.
	 * @return void
	 */
	public function delete_post_index( $post_id ) {
		// Pages are stored as 'page'; remove both, with their vectors.
		$this->remove_index_row( 'post', (int) $post_id );
		$this->remove_index_row( 'page', (int) $post_id );
	}

	/**
	 * Delete the index for a comment.
	 *
	 * This method deletes the index entry for a specific comment by its ID.
	 *
	 * @param int $comment_id Comment ID to delete from the index.
	 * @return void
	 */
	public function delete_comment_index( $comment_id ) {
		global $wpdb;
		$tables = $this->database->get_tables();
		$content_index_table = $tables['content_index'];

		$wpdb->delete(
			$content_index_table,
			array(
				'object_type' => 'comment',
				'object_id' => $comment_id,
			),
			array( '%s', '%d' )
		);
	}

	/**
	 * Delete the index for a product.
	 *
	 * This method deletes the index entry for a specific product by its ID.
	 *
	 * @param int $product_id Product ID to delete from the index.
	 * @return void
	 */
	public function delete_product_index( $product_id ) {
		global $wpdb;
		$tables = $this->database->get_tables();
		$content_index_table = $tables['content_index'];

		$wpdb->delete(
			$content_index_table,
			array(
				'object_type' => 'product',
				'object_id' => $product_id,
			),
			array( '%s', '%d' )
		);
	}

	/**
	 * Index pages in batches.
	 *
	 * This method retrieves all published pages and indexes them in batches.
	 *
	 * @return void
	 */
	private function index_pages() {
		$args = array(
			'post_type' => 'page',
			'post_status' => 'publish',
			'posts_per_page' => -1,
			'fields' => 'ids',
		);

		$page_ids = get_posts($args);
		$total_pages = count($page_ids);

		for ( $i = 0; $i < $total_pages; $i += $this->batch_size ) {
			$batch = array_slice($page_ids, $i, $this->batch_size);
			foreach ( $batch as $page_id ) {
				$this->index_page($page_id);
			}
		}
	}

	/**
	 * Index template pages.
	 *
	 * This method indexes page templates and their content.
	 *
	 * @return void
	 */
	private function index_template_pages() {
		// Pages that USE a theme template are indexed by index_page() from their
		// rendered HTML (real URL, template parts, DB prices). Indexing them here
		// again overwrote that row with a raw-PHP parse on every run (double
		// embedding cost and worse text), so this loop only handles template files.
		$used = $this->get_used_page_templates();

		/**
		 * Index theme template files that no published page uses. They have no
		 * URL and only contain placeholder/hard-coded text, so this is off by default.
		 *
		 * @param bool $index Default false.
		 */
		$index_unused = (bool) apply_filters( 'askany_index_unused_template_files', false );

		$templates = wp_get_theme()->get_page_templates();
		foreach ( $templates as $template_file => $template_name ) {
			if ( isset( $used[ $template_file ] ) || ! $index_unused || $this->is_template_skipped( $template_file ) ) {
				// Drop the old URL-less copy left by earlier versions.
				$this->remove_index_row( 'template_page', crc32( $template_file ) );
				continue;
			}
			$this->index_template_page( $template_file, $template_name );
		}
	}

	/**
	 * Theme template files used by at least one published page.
	 *
	 * @return array<string,int[]> template file => page IDs
	 */
	private function get_used_page_templates() {
		global $wpdb;
		// phpcs:ignore WordPress.DB.DirectDatabaseQuery
		$rows = $wpdb->get_results(
			"SELECT pm.post_id, pm.meta_value FROM {$wpdb->postmeta} pm
			 JOIN {$wpdb->posts} p ON p.ID = pm.post_id
			 WHERE pm.meta_key = '_wp_page_template' AND pm.meta_value NOT IN ('', 'default')
			   AND p.post_type = 'page' AND p.post_status = 'publish'"
		);
		$used = array();
		foreach ( (array) $rows as $row ) {
			$used[ $row->meta_value ][] = (int) $row->post_id;
		}
		return $used;
	}

	/**
	 * Classic theme template file a page uses ('' when none / block theme).
	 *
	 * @param int $page_id Page ID.
	 * @return string
	 */
	private function get_classic_template( $page_id ) {
		$template = (string) get_page_template_slug( $page_id );
		if ( '' === $template || 'default' === $template || ! preg_match( '/\.php$/i', $template ) ) {
			return '';
		}
		return $template;
	}

	/**
	 * Template files the site never wants in the knowledge base
	 * (e.g. buyer-only or applicant pages).
	 *
	 * @param string $template_file Template file.
	 * @param int    $page_id       Page using it (0 for the file itself).
	 * @return bool
	 */
	private function is_template_skipped( $template_file, $page_id = 0 ) {
		$skip = (array) apply_filters( 'askany_template_index_skip', array(), $page_id );
		return in_array( $template_file, $skip, true ) || in_array( basename( $template_file ), $skip, true );
	}

	/**
	 * Delete one index row and its vectors.
	 *
	 * @param string     $object_type Object type.
	 * @param int|string $object_id   Object ID.
	 * @return void
	 */
	private function remove_index_row( $object_type, $object_id ) {
		global $wpdb;
		$tables = $this->database->get_tables();
		$table  = $tables['content_index'];
		$ids    = $wpdb->get_col( $wpdb->prepare( "SELECT id FROM {$table} WHERE object_type = %s AND object_id = %s", $object_type, $object_id ) ); // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
		foreach ( $ids as $id ) {
			$wpdb->delete( $wpdb->prefix . 'askany_embeddings', array( 'content_id' => (int) $id ), array( '%d' ) );
			$wpdb->delete( $table, array( 'id' => (int) $id ), array( '%d' ) );
		}
	}

	/**
	 * The crawler, used to render template pages over HTTP.
	 *
	 * @return ExternalSourcesHandler|null
	 */
	private function get_external_handler() {
		if ( null === $this->external_handler ) {
			$this->external_handler = class_exists( __NAMESPACE__ . '\\ExternalSourcesHandler' )
				? new ExternalSourcesHandler( $this->database )
				: false;
		}
		return $this->external_handler ? $this->external_handler : null;
	}

	/**
	 * Cron/deferred entry point: render one template page now (no time budget).
	 *
	 * @param int $page_id Page ID.
	 * @return void
	 */
	public function render_template_page( $page_id ) {
		$this->index_page( (int) $page_id, true );
	}

	/**
	 * Index a page built from a classic theme template.
	 *
	 * Order of preference:
	 * 1. An External Source already crawls this URL → skip (no duplicate).
	 * 2. Render the live page over HTTP (X-AskAny-Crawl: 1) and index the
	 *    clean text: template parts, DB-driven prices and real copy included.
	 * 3. Fallback: the old static parse of the template PHP file.
	 *
	 * @param \WP_Post $page     Page.
	 * @param string   $template Template file.
	 * @param bool     $force    Ignore the per-request render time budget.
	 * @return void
	 */
	private function index_template_backed_page( $page, $template, $force = false ) {
		$page_id = (int) $page->ID;
		$url     = (string) get_permalink( $page_id );

		if ( $this->is_template_skipped( $template, $page_id ) ) {
			$this->remove_index_row( 'page', $page_id );
			return;
		}

		$handler = $this->get_external_handler();

		/**
		 * Skip template pages an External Source already indexes (default true).
		 *
		 * @param bool $skip    Default true.
		 * @param int  $page_id Page ID.
		 */
		if ( $handler && $url && apply_filters( 'askany_skip_pages_covered_by_external_source', true, $page_id ) && $handler->covers_url( $url ) ) {
			$this->remove_index_row( 'page', $page_id );
			return;
		}

		/**
		 * Render template pages over HTTP for indexing (default true).
		 *
		 * @param bool $render  Default true.
		 * @param int  $page_id Page ID.
		 */
		$render = $handler && $url && apply_filters( 'askany_render_template_pages', true, $page_id );

		if ( $render ) {
			$budget = (float) apply_filters( 'askany_template_render_budget', 20 );
			if ( ! $force && $budget > 0 && $this->render_seconds >= $budget ) {
				// Out of time in this request: keep the current row (mark it seen so
				// a full re-index does not delete it) and render it in the background.
				if ( ! wp_next_scheduled( 'askany_render_template_page', array( $page_id ) ) ) {
					wp_schedule_single_event( time() + 30 + wp_rand( 0, 120 ), 'askany_render_template_page', array( $page_id ) );
				}
				if ( $this->touch_index_row( 'page', $page_id ) ) {
					return;
				}
				$render = false; // No row yet: store the static parse now, render later.
			}
		}

		if ( $render ) {
			$start    = microtime( true );
			$rendered = $handler->fetch_page_text( $url );
			$this->render_seconds += microtime( true ) - $start;

			$min = (int) apply_filters( 'askany_template_render_min_chars', 200 );
			if ( $rendered && strlen( trim( $rendered['content'] ) ) >= $min ) {
				$parts = array( $page->post_title, trim( $rendered['content'] ) );
				$desc  = get_post_meta( $page_id, '_yoast_wpseo_metadesc', true );
				if ( $desc ) {
					$parts[] = 'SEO Description: ' . $desc;
				}
				// Keep line breaks (sections, list items, table rows) — they help chunking.
				$text = implode( "\n", $parts );
				$text = preg_replace( '/[ \t\x{00A0}]+/u', ' ', $text );
				$text = trim( (string) preg_replace( '/ *\n\s*/u', "\n", (string) $text ) );
				$this->store_content( 'page', $page_id, $text, $url, $page->post_title );
				return;
			}
		}

		// Fallback: static parse of the template file (stored with the real permalink).
		$this->index_page_with_template( $page_id );
	}

	/**
	 * Mark an existing index row as seen in this run.
	 *
	 * @param string $object_type Object type.
	 * @param int    $object_id   Object ID.
	 * @return bool True when a row exists.
	 */
	private function touch_index_row( $object_type, $object_id ) {
		global $wpdb;
		$tables = $this->database->get_tables();
		// phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared
		$id = (int) $wpdb->get_var( $wpdb->prepare( "SELECT id FROM {$tables['content_index']} WHERE object_type = %s AND object_id = %s", $object_type, $object_id ) );
		if ( ! $id ) {
			return false;
		}
		$wpdb->query( $wpdb->prepare( "UPDATE {$tables['content_index']} SET updated_at = %s WHERE id = %d", current_time( 'mysql' ), $id ) );
		// phpcs:enable
		return true;
	}

	/**
	 * Get all documentation, knowledgebase, and custom post types registered on the site.
	 *
	 * @return array List of post type slugs.
	 */
	private function get_all_doc_and_cpt_types() {
		$known_types = array(
			'docs', 'doc', 'betterdocs', 'betterdocs_doc',
			'documentation', 'knowledge_base', 'knowledgebase', 'kb',
			'ht_kb', 'wedocs', 'wiki', 'faq', 'faqs', 'manual', 'manuals', 'guide'
		);

		$registered_cpts = get_post_types( array( 'public' => true, '_builtin' => false ), 'names' );
		$all_types       = array_unique( array_merge( $known_types, array_values( $registered_cpts ) ) );
		$excluded        = array( 'product', 'attachment', 'revision', 'nav_menu_item', 'custom_css', 'customize_changeset' );

		$final_types = array();
		foreach ( $all_types as $pt ) {
			// Public, viewable types are already indexed by index_posts(); indexing
			// them again here stored and embedded every item twice.
			if ( ! in_array( $pt, $excluded, true ) && post_type_exists( $pt ) && ! is_post_type_viewable( $pt ) ) {
				$final_types[] = $pt;
			}
		}

		return $final_types;
	}

	/**
	 * Index documentation pages.
	 *
	 * This method indexes documentation-related content and custom post types.
	 *
	 * @return void
	 */
	private function index_docs_pages() {
		// Index all documentation & custom post types including BetterDocs
		$doc_post_types = $this->get_all_doc_and_cpt_types();
		
		foreach ( $doc_post_types as $post_type ) {
			if ( post_type_exists($post_type) ) {
				$args = array(
					'post_type' => $post_type,
					'post_status' => 'publish',
					'posts_per_page' => -1,
					'fields' => 'ids',
				);

				$doc_ids = get_posts($args);
				foreach ( $doc_ids as $doc_id ) {
					$this->index_docs_page($doc_id);
				}
			}
		}
		
		// Index BetterDocs categories and tags if plugin is active
		if ( class_exists('BetterDocs') || taxonomy_exists('doc_category') ) {
			$this->index_betterdocs_taxonomies();
		}
	}

	/**
	 * Index BetterDocs taxonomies (categories, tags, knowledge bases).
	 *
	 * @return void
	 */
	private function index_betterdocs_taxonomies() {
		// Index doc categories
		if ( taxonomy_exists('doc_category') ) {
			$doc_categories = get_terms(array(
				'taxonomy' => 'doc_category',
				'hide_empty' => false,
			));

			foreach ( $doc_categories as $category ) {
				$content = array(
					$category->name,
					$category->description,
					$category->slug,
				);

				// Get category icon if available
				$icon = get_term_meta($category->term_id, 'doc_category_image-id', true);
				if ( $icon ) {
					$content[] = "Has icon";
				}

				// Get category order
				$order = get_term_meta($category->term_id, 'doc_category_order', true);
				if ( $order ) {
					$content[] = "Order: " . $order;
				}

				$prepared_content = $this->prepare_content(implode("\n", $content));
				if ( ! empty($prepared_content) ) {
					$this->store_content('doc_category', $category->term_id, $prepared_content, get_term_link($category), $category->name);
				}
			}
		}

		// Index doc tags
		if ( taxonomy_exists('doc_tag') ) {
			$doc_tags = get_terms(array(
				'taxonomy' => 'doc_tag',
				'hide_empty' => false,
			));

			foreach ( $doc_tags as $tag ) {
				$content = array(
					$tag->name,
					$tag->description,
					$tag->slug,
				);

				$prepared_content = $this->prepare_content(implode("\n", $content));
				if ( ! empty($prepared_content) ) {
					$this->store_content('doc_tag', $tag->term_id, $prepared_content, get_term_link($tag), $tag->name);
				}
			}
		}

		// Index knowledge bases
		if ( taxonomy_exists('knowledge_base') ) {
			$knowledge_bases = get_terms(array(
				'taxonomy' => 'knowledge_base',
				'hide_empty' => false,
			));

			foreach ( $knowledge_bases as $kb ) {
				$content = array(
					$kb->name,
					$kb->description,
					$kb->slug,
				);

				$prepared_content = $this->prepare_content(implode("\n", $content));
				if ( ! empty($prepared_content) ) {
					$this->store_content('knowledge_base', $kb->term_id, $prepared_content, get_term_link($kb), $kb->name);
				}
			}
		}
	}

	/**
	 * Index product categories.
	 *
	 * This method indexes WooCommerce product categories and their descriptions.
	 *
	 * @return void
	 */
	public function index_product_categories() {
		$categories = get_terms(array(
			'taxonomy' => 'product_cat',
			'hide_empty' => false,
		));

		foreach ( $categories as $category ) {
			$this->index_product_category($category);
		}
	}

	/**
	 * Index product tags.
	 *
	 * This method indexes WooCommerce product tags and their descriptions.
	 *
	 * @return void
	 */
	public function index_product_tags() {
		$tags = get_terms(array(
			'taxonomy' => 'product_tag',
			'hide_empty' => false,
		));

		foreach ( $tags as $tag ) {
			$this->index_product_tag($tag);
		}
	}

	/**
	 * Index product attributes.
	 *
	 * This method indexes WooCommerce global product attributes and their terms.
	 *
	 * @return void
	 */
	public function index_product_attributes() {
		$attribute_taxonomies = wc_get_attribute_taxonomies();
		if ( ! empty( $attribute_taxonomies ) ) {
			foreach ( $attribute_taxonomies as $tax ) {
				$taxonomy_name = wc_attribute_taxonomy_name( $tax->attribute_name );
				$terms = get_terms( array(
					'taxonomy'   => $taxonomy_name,
					'hide_empty' => false,
				) );
				
				if ( ! is_wp_error( $terms ) && ! empty( $terms ) ) {
					foreach ( $terms as $term ) {
						$this->index_product_attribute( $term, $tax->attribute_label );
					}
				}
			}
		}
	}

	/**
	 * Index a product attribute term.
	 *
	 * @param WP_Term $term  Product attribute term.
	 * @param string  $label Attribute label.
	 * @return void
	 */
	private function index_product_attribute( $term, $label ) {
		$content = array(
			$label . ': ' . $term->name,
			$term->description,
			$term->slug,
		);

		$prepared_content = $this->prepare_content( implode( "\n", $content ) );
		if ( empty( $prepared_content ) ) {
			return;
		}

		$this->store_content( 'product_attribute', $term->term_id, $prepared_content, get_term_link( $term ), $term->name );
	}

	/**
	 * Index custom fields across the website.
	 *
	 * This method indexes custom fields and meta data from posts, pages, and products.
	 *
	 * @return void
	 */
	private function index_custom_fields() {
		global $wpdb;

		// Get all custom fields with meaningful content
		$custom_fields = $wpdb->get_results("
			SELECT DISTINCT meta_key, meta_value, post_id 
			FROM {$wpdb->postmeta} 
			WHERE meta_key NOT LIKE '\_%' 
			AND meta_value != '' 
			AND LENGTH(meta_value) > 10
			LIMIT 1000
		");

		foreach ( $custom_fields as $field ) {
			$this->index_custom_field($field);
		}
	}

	/**
	 * Index navigation menus.
	 *
	 * This method indexes WordPress navigation menus and their items.
	 *
	 * @return void
	 */
	public function index_menus() {
		$menus = wp_get_nav_menus();

		foreach ( $menus as $menu ) {
			$this->index_menu($menu);
		}
	}

	/**
	 * Index widgets.
	 *
	 * This method indexes active widgets and their content.
	 *
	 * @return void
	 */
	private function index_widgets() {
		$sidebars = wp_get_sidebars_widgets();

		foreach ( $sidebars as $sidebar_id => $widgets ) {
			if ( is_array($widgets) ) {
				foreach ( $widgets as $widget_id ) {
					$this->index_widget($widget_id, $sidebar_id);
				}
			}
		}
	}

	/**
	 * Index site XML sitemaps to discover all published URLs.
	 *
	 * @return void
	 */
	private function index_sitemaps() {
		$sitemap_urls = array(
			home_url( '/wp-sitemap.xml' ),
			home_url( '/sitemap_index.xml' ),
			home_url( '/sitemap.xml' ),
		);

		$discovered_urls = array();

		foreach ( $sitemap_urls as $sitemap_url ) {
			$response = wp_remote_get( $sitemap_url, array(
				'timeout'   => 8,
				'sslverify' => false,
			) );

			if ( is_wp_error( $response ) || wp_remote_retrieve_response_code( $response ) !== 200 ) {
				continue;
			}

			$body = wp_remote_retrieve_body( $response );
			if ( empty( $body ) ) {
				continue;
			}

			if ( preg_match_all( '/<loc>(.*?)<\/loc>/i', $body, $matches ) ) {
				foreach ( $matches[1] as $url ) {
					$clean_url = esc_url_raw( trim( $url ) );
					if ( $clean_url && strpos( $clean_url, '.xml' ) === false ) {
						$discovered_urls[] = $clean_url;
					}
				}
			}
		}

		$discovered_urls = array_unique( $discovered_urls );
		if ( empty( $discovered_urls ) ) {
			return;
		}

		$sitemap_content = array( "Site Sitemap Permalinks & Page Hierarchy:" );
		foreach ( $discovered_urls as $url ) {
			$path  = parse_url( $url, PHP_URL_PATH );
			$slug  = trim( (string) $path, '/' );
			$label = ! empty( $slug ) ? ucwords( str_replace( array( '-', '/' ), ' ', $slug ) ) : 'Home';

			$sitemap_content[] = sprintf( "- %s: %s", $label, $url );
		}

		$prepared = $this->prepare_content( implode( "\n", $sitemap_content ) );
		if ( ! empty( $prepared ) ) {
			$this->store_content( 'sitemap_url', 1, $prepared, home_url( '/' ), 'Site Sitemap' );
		}
	}

	/**
	 * Index a single page.
	 *
	 * @param int $page_id Page ID to index.
	 * @return void
	 */
	private function index_page( $page_id, $force_render = false ) {
		global $post;

		$page = get_post($page_id);
		if ( ! $page || $page->post_status !== 'publish' || ! empty( $page->post_password ) ) {
			return;
		}

		// Classic theme template (e.g. landing/pricing pages): index the rendered page.
		$classic_template = $this->get_classic_template( $page_id );
		if ( $classic_template ) {
			$this->index_template_backed_page( $page, $classic_template, $force_render );
			return;
		}

		// Set global post for Elementor extraction
		$old_post = $post;
		$post = $page;

		$content = array(
			$page->post_title,
			$page->post_content,
			$page->post_excerpt,
		);

		// Add page template info
		$template = get_page_template_slug($page_id);
		if ( $template ) {
			$content[] = "Template: " . $template;
		}
		
		// Add Elementor meta description if available
		$elementor_settings = get_post_meta( $page_id, '_elementor_page_settings', true );
		if ( ! empty( $elementor_settings['post_excerpt'] ) ) {
			$content[] = $elementor_settings['post_excerpt'];
		}
		
		// Add custom fields that might contain important info
		$custom_fields = array(
			'_yoast_wpseo_metadesc' => 'SEO Description',
			'_yoast_wpseo_title' => 'SEO Title',
			'page_subtitle' => 'Subtitle',
			'page_description' => 'Description',
		);
		
		foreach ( $custom_fields as $field => $label ) {
			$meta_value = get_post_meta( $page_id, $field, true );
			if ( $meta_value ) {
				$content[] = $label . ": " . $meta_value;
			}
		}

		$prepared_content = $this->prepare_content(implode("\n", $content));
		
		// Restore global post
		$post = $old_post;
		
		if ( empty($prepared_content) ) {
			return;
		}

		$this->store_content('page', $page_id, $prepared_content, get_permalink($page_id), $page->post_title);
	}

	/**
	 * Index a template page.
	 *
	 * @param string $template_file Template file name.
	 * @param string $template_name Template display name.
	 * @return void
	 */
	private function index_template_page( $template_file, $template_name ) {
		$content = array(
			$template_name,
			"Template file: " . $template_file,
		);

		// Try to read template file content from theme directories
		$template_paths = array(
			get_template_directory() . '/' . $template_file,
			get_stylesheet_directory() . '/' . $template_file,
			// Check in templates subdirectory (common location)
			get_template_directory() . '/templates/' . $template_file,
			get_stylesheet_directory() . '/templates/' . $template_file,
		);
		
		$template_path = '';
		foreach ( $template_paths as $path ) {
			if ( file_exists( $path ) ) {
				$template_path = $path;
				break;
			}
		}

		if ( ! empty( $template_path ) && file_exists( $template_path ) ) {
			$file_content = file_get_contents( $template_path );
			
			// Extract PHP doc comments and Template Name
			preg_match_all('/\/\*\*(.*?)\*\//s', $file_content, $doc_matches);
			if ( ! empty( $doc_matches[1] ) ) {
				foreach ( $doc_matches[1] as $doc_comment ) {
					// Clean up doc comment
					$doc_comment = preg_replace('/^\s*\*\s*/m', '', $doc_comment);
					$content[] = trim( $doc_comment );
				}
			}
			
			// Extract all translatable strings
			preg_match_all('/esc_html_e\([\'"]([^\'"]+)[\'"]/i', $file_content, $i18n_matches);
			if ( ! empty( $i18n_matches[1] ) ) {
				$content = array_merge( $content, $i18n_matches[1] );
			}
			
			preg_match_all('/esc_html__\([\'"]([^\'"]+)[\'"]/i', $file_content, $i18n_matches2);
			if ( ! empty( $i18n_matches2[1] ) ) {
				$content = array_merge( $content, $i18n_matches2[1] );
			}
			
			preg_match_all('/esc_attr_e\([\'"]([^\'"]+)[\'"]/i', $file_content, $i18n_matches3);
			if ( ! empty( $i18n_matches3[1] ) ) {
				$content = array_merge( $content, $i18n_matches3[1] );
			}
			
			preg_match_all('/esc_attr__\([\'"]([^\'"]+)[\'"]/i', $file_content, $i18n_matches4);
			if ( ! empty( $i18n_matches4[1] ) ) {
				$content = array_merge( $content, $i18n_matches4[1] );
			}
			
			// Extract content from esc_html() wrapping strings
			preg_match_all('/esc_html\([\'"]([^\'"]+)[\'"]\)/i', $file_content, $esc_html_matches);
			if ( ! empty( $esc_html_matches[1] ) ) {
				$content = array_merge( $content, $esc_html_matches[1] );
			}
			
			// Extract HTML content (text between tags)
			// Create a version with PHP short tags processed
			$html_content = $file_content;
			
			// Remove PHP blocks but keep their echo output patterns
			$html_content = preg_replace('/<\?php\s+echo\s+esc_html\([\'"]([^\'"]+)[\'"]\)\s*;\s*\?>/i', '$1', $html_content);
			$html_content = preg_replace('/<\?php.*?\?>/s', ' ', $html_content);
			
			// Extract text from HTML using DOMDocument
			$dom = new \DOMDocument();
			libxml_use_internal_errors(true); // Suppress HTML5 warnings
			@$dom->loadHTML( '<?xml encoding="UTF-8">' . $html_content, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD );
			libxml_clear_errors();
			
			// Extract text from all relevant elements
			$text_elements = array( 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'li', 'span', 'div', 'a', 'button', 'label', 'td', 'th' );
			foreach ( $text_elements as $tag ) {
				$elements = $dom->getElementsByTagName( $tag );
				foreach ( $elements as $element ) {
					$text = trim( $element->textContent );
					// Filter out empty strings, very short strings, and common noise
					if ( ! empty( $text ) && strlen( $text ) > 2 && ! preg_match('/^[\d\s\$\%\.\,\-]+$/', $text) ) {
						$content[] = $text;
					}
				}
			}
			
			// Extract class names that might be descriptive (like "hero-title", "feature-description")
			preg_match_all('/class=["\']([^"\']*(?:title|heading|description|content|text|feature|benefit|section)[^"\']*)["\']/', $file_content, $class_matches);
			if ( ! empty( $class_matches[1] ) ) {
				foreach ( $class_matches[1] as $class_name ) {
					// Convert class names to readable text (e.g., "hero-title" -> "hero title")
					$readable = str_replace( array('-', '_'), ' ', $class_name );
					if ( strlen( $readable ) > 5 ) {
						$content[] = "Section: " . $readable;
					}
				}
			}
			
			// Extract alt text from images
			preg_match_all('/alt=["\']([^"\']+)["\']/', $file_content, $alt_matches);
			if ( ! empty( $alt_matches[1] ) ) {
				foreach ( $alt_matches[1] as $alt_text ) {
					if ( strlen( $alt_text ) > 3 ) {
						$content[] = "Image: " . $alt_text;
					}
				}
			}
			
			// Extract aria-label attributes for accessibility content
			preg_match_all('/aria-label=["\']([^"\']+)["\']/', $file_content, $aria_matches);
			if ( ! empty( $aria_matches[1] ) ) {
				$content = array_merge( $content, $aria_matches[1] );
			}
		}

		// Remove duplicates and empty values
		$content = array_unique( array_filter( $content ) );
		
		$prepared_content = $this->prepare_content(implode("\n", $content));
		if ( empty($prepared_content) ) {
			return;
		}

		$this->store_content('template_page', crc32($template_file), $prepared_content, '', $template_name);
	}
	
	/**
	 * Index a page that uses a custom template
	 *
	 * @param int $page_id Page ID
	 * @return void
	 */
	private function index_page_with_template( $page_id ) {
		global $post;
		
		$page = get_post( $page_id );
		if ( ! $page || $page->post_status !== 'publish' ) {
			return;
		}
		
		// Set global post for Elementor extraction
		$old_post = $post;
		$post = $page;
		
		$template_file = get_page_template_slug( $page_id );
		
		$content = array(
			$page->post_title,
			$page->post_content,
			$page->post_excerpt,
			"Template: " . $template_file,
		);
		
		// Get template content from theme directories
		$template_paths = array(
			get_template_directory() . '/' . $template_file,
			get_stylesheet_directory() . '/' . $template_file,
			get_template_directory() . '/templates/' . $template_file,
			get_stylesheet_directory() . '/templates/' . $template_file,
		);
		
		foreach ( $template_paths as $template_path ) {
			if ( file_exists( $template_path ) ) {
				$file_content = file_get_contents( $template_path );
				
				// Extract all translatable strings from the template
				preg_match_all('/esc_html_e\([\'"]([^\'"]+)[\'"]/i', $file_content, $i18n_matches);
				if ( ! empty( $i18n_matches[1] ) ) {
					$content = array_merge( $content, $i18n_matches[1] );
				}
				
				preg_match_all('/esc_html__\([\'"]([^\'"]+)[\'"]/i', $file_content, $i18n_matches2);
				if ( ! empty( $i18n_matches2[1] ) ) {
					$content = array_merge( $content, $i18n_matches2[1] );
				}
				
				preg_match_all('/esc_attr_e\([\'"]([^\'"]+)[\'"]/i', $file_content, $i18n_matches3);
				if ( ! empty( $i18n_matches3[1] ) ) {
					$content = array_merge( $content, $i18n_matches3[1] );
				}
				
				preg_match_all('/esc_html\([\'"]([^\'"]+)[\'"]\)/i', $file_content, $esc_html_matches);
				if ( ! empty( $esc_html_matches[1] ) ) {
					$content = array_merge( $content, $esc_html_matches[1] );
				}
				
				// Extract visible text content from HTML
				$html_content = preg_replace('/<\?php\s+echo\s+esc_html\([\'"]([^\'"]+)[\'"]\)\s*;\s*\?>/i', '$1', $file_content);
				$html_content = preg_replace('/<\?php.*?\?>/s', ' ', $html_content);
				
				$dom = new \DOMDocument();
				libxml_use_internal_errors(true);
				@$dom->loadHTML( '<?xml encoding="UTF-8">' . $html_content, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD );
				libxml_clear_errors();
				
				$text_elements = array( 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'li', 'span', 'a', 'button', 'label', 'td', 'th' );
				foreach ( $text_elements as $tag ) {
					$elements = $dom->getElementsByTagName( $tag );
					foreach ( $elements as $element ) {
						$text = trim( $element->textContent );
						if ( ! empty( $text ) && strlen( $text ) > 2 && ! preg_match('/^[\d\s\$\%\.\,\-]+$/', $text) ) {
							$content[] = $text;
						}
					}
				}
				
				// Extract alt text and aria-labels
				preg_match_all('/alt=["\']([^"\']+)["\']/', $file_content, $alt_matches);
				if ( ! empty( $alt_matches[1] ) ) {
					foreach ( $alt_matches[1] as $alt_text ) {
						if ( strlen( $alt_text ) > 3 ) {
							$content[] = $alt_text;
						}
					}
				}
				
				preg_match_all('/aria-label=["\']([^"\']+)["\']/', $file_content, $aria_matches);
				if ( ! empty( $aria_matches[1] ) ) {
					$content = array_merge( $content, $aria_matches[1] );
				}
				
				// Extract section/feature descriptions from class names
				preg_match_all('/class=["\']([^"\']*(?:title|heading|description|content|text|feature|benefit)[^"\']*)["\']/', $file_content, $class_matches);
				if ( ! empty( $class_matches[1] ) ) {
					foreach ( $class_matches[1] as $class_name ) {
						$readable = str_replace( array('-', '_'), ' ', $class_name );
						if ( strlen( $readable ) > 5 ) {
							$content[] = "Section: " . $readable;
						}
					}
				}
				
				break;
			}
		}
		
		// Add custom fields
		$custom_fields = array(
			'_yoast_wpseo_metadesc' => 'SEO Description',
			'_yoast_wpseo_title' => 'SEO Title',
		);
		
		foreach ( $custom_fields as $field => $label ) {
			$meta_value = get_post_meta( $page_id, $field, true );
			if ( $meta_value ) {
				$content[] = $label . ": " . $meta_value;
			}
		}
		
		// Remove duplicates
		$content = array_unique( array_filter( $content ) );
		
		$prepared_content = $this->prepare_content( implode( "\n", $content ) );
		
		// Restore global post
		$post = $old_post;
		
		if ( empty( $prepared_content ) ) {
			return;
		}
		
		$this->store_content( 'page', $page_id, $prepared_content, get_permalink( $page_id ), $page->post_title );
	}

	/**
	 * Index a documentation page.
	 *
	 * @param int $doc_id Documentation page ID.
	 * @return void
	 */
	private function index_docs_page( $doc_id ) {
		$doc = get_post($doc_id);
		if ( ! $doc || $doc->post_status !== 'publish' ) {
			return;
		}

		$content = array(
			$doc->post_title,
			$doc->post_content,
			$doc->post_excerpt,
		);

		// Add BetterDocs categories
		$doc_categories = wp_get_post_terms($doc_id, 'doc_category');
		if ( ! empty($doc_categories) && ! is_wp_error($doc_categories) ) {
			foreach ( $doc_categories as $category ) {
				$content[] = "Category: " . $category->name;
				if ( ! empty($category->description) ) {
					$content[] = $category->description;
				}
			}
		}

		// Add BetterDocs tags
		$doc_tags = wp_get_post_terms($doc_id, 'doc_tag');
		if ( ! empty($doc_tags) && ! is_wp_error($doc_tags) ) {
			foreach ( $doc_tags as $tag ) {
				$content[] = "Tag: " . $tag->name;
			}
		}

		// Add knowledge base taxonomy
		$knowledge_bases = wp_get_post_terms($doc_id, 'knowledge_base');
		if ( ! empty($knowledge_bases) && ! is_wp_error($knowledge_bases) ) {
			foreach ( $knowledge_bases as $kb ) {
				$content[] = "Knowledge Base: " . $kb->name;
			}
		}

		// Add custom fields that might contain documentation metadata
		$meta_fields = array(
			'doc_category' => 'Category',
			'doc_tags' => 'Tags',
			'doc_difficulty' => 'Difficulty',
			'doc_version' => 'Version',
			// View/reaction counters left out: they change on every visit and
			// forced the doc to be embedded again each time.
		);
		
		foreach ( $meta_fields as $field => $label ) {
			$meta_value = get_post_meta($doc_id, $field, true);
			if ( $meta_value ) {
				$content[] = $label . ": " . $meta_value;
			}
		}

		// Add related docs if available
		$related_docs = get_post_meta($doc_id, '_betterdocs_related_docs', true);
		if ( ! empty($related_docs) && is_array($related_docs) ) {
			foreach ( $related_docs as $related_id ) {
				$related_post = get_post($related_id);
				if ( $related_post ) {
					$content[] = "Related: " . $related_post->post_title;
				}
			}
		}

		$prepared_content = $this->prepare_content(implode("\n", $content));
		if ( empty($prepared_content) ) {
			return;
		}

		$this->store_content('docs', $doc_id, $prepared_content, get_permalink($doc_id), $doc->post_title);
	}

	/**
	 * Index a product category.
	 *
	 * @param WP_Term $category Product category term.
	 * @return void
	 */
	private function index_product_category( $category ) {
		$content = array(
			$category->name,
			$category->description,
			$category->slug,
		);

		$prepared_content = $this->prepare_content(implode("\n", $content));
		if ( empty($prepared_content) ) {
			return;
		}

		$this->store_content('product_category', $category->term_id, $prepared_content, get_term_link($category), $category->name);
	}

	/**
	 * Index a product tag.
	 *
	 * @param WP_Term $tag Product tag term.
	 * @return void
	 */
	private function index_product_tag( $tag ) {
		$content = array(
			$tag->name,
			$tag->description,
			$tag->slug,
		);

		$prepared_content = $this->prepare_content(implode("\n", $content));
		if ( empty($prepared_content) ) {
			return;
		}

		$this->store_content('product_tag', $tag->term_id, $prepared_content, get_term_link($tag), $tag->name);
	}

	/**
	 * Index a custom field.
	 *
	 * @param object $field Custom field data.
	 * @return void
	 */
	private function index_custom_field( $field ) {
		$content = array(
			$field->meta_key,
			$field->meta_value,
		);

		$prepared_content = $this->prepare_content(implode(": ", $content));
		if ( empty($prepared_content) ) {
			return;
		}

		$post_url = get_permalink($field->post_id);
		$this->store_content('custom_field', $field->post_id . '_' . crc32($field->meta_key), $prepared_content, $post_url, $field->meta_key);
	}

	/**
	 * Index a navigation menu.
	 *
	 * @param WP_Term $menu Menu term object.
	 * @return void
	 */
	private function index_menu( $menu ) {
		$menu_items = wp_get_nav_menu_items($menu->term_id);
		if ( empty( $menu_items ) ) {
			return;
		}

		$content = array( sprintf( "Navigation Menu: %s", $menu->name ) );

		foreach ( $menu_items as $item ) {
			$title = ! empty( $item->title ) ? trim( $item->title ) : '';
			$url   = ! empty( $item->url ) ? esc_url_raw( $item->url ) : '';

			if ( empty( $title ) && empty( $url ) ) {
				continue;
			}

			if ( $url ) {
				$entry = sprintf( "- %s: %s", $title ? $title : $url, $url );
			} else {
				$entry = sprintf( "- %s", $title );
			}

			if ( ! empty( $item->description ) ) {
				$entry .= sprintf( " (%s)", trim( $item->description ) );
			}

			$content[] = $entry;
		}

		$prepared_content = $this->prepare_content( implode( "\n", $content ) );
		if ( empty( $prepared_content ) ) {
			return;
		}

		$first_url = '';
		foreach ( $menu_items as $item ) {
			if ( ! empty( $item->url ) && strpos( $item->url, 'http' ) === 0 ) {
				$first_url = esc_url_raw( $item->url );
				break;
			}
		}

		$this->store_content('menu', $menu->term_id, $prepared_content, $first_url, $menu->name);
	}

	/**
	 * Index a widget.
	 *
	 * @param string $widget_id Widget ID.
	 * @param string $sidebar_id Sidebar ID.
	 * @return void
	 */
	private function index_widget( $widget_id, $sidebar_id ) {
		global $wp_registered_widgets;

		if ( ! isset($wp_registered_widgets[$widget_id]) ) {
			return;
		}

		$widget = $wp_registered_widgets[$widget_id];
		$content = array(
			$widget['name'],
			"Sidebar: " . $sidebar_id,
		);

		// Try to get widget settings
		$widget_base = $widget['callback'][0]->id_base ?? '';
		if ( $widget_base ) {
			$widget_options = get_option('widget_' . $widget_base);
			// Only this widget's own settings (every instance of the widget type
			// was dumped for each placed widget, creating near-duplicate rows).
			$number = $widget['params'][0]['number'] ?? null;
			$option = ( is_array( $widget_options ) && null !== $number && isset( $widget_options[ $number ] ) ) ? $widget_options[ $number ] : null;
			if ( is_array( $option ) ) {
				foreach ( $option as $key => $value ) {
					if ( is_string( $value ) && strlen( $value ) > 5 ) {
						$content[] = ( 'title' === $key ? 'Title' : $key ) . ': ' . $value;
					}
				}
			}
		}

		$prepared_content = $this->prepare_content(implode("\n", $content));
		if ( empty($prepared_content) ) {
			return;
		}

		$this->store_content('widget', crc32($widget_id), $prepared_content, '');
	}

	/**
	 * Store content in the database.
	 *
	 * @param string $object_type Type of object being indexed.
	 * @param mixed  $object_id   ID of the object.
	 * @param string $content     Prepared content.
	 * @param string $url         URL of the content.
	 * @return void
	 */
	private function store_content( $object_type, $object_id, $content, $url, $title = '' ) {
		global $wpdb;
		$tables = $this->database->get_tables();
		$content_index_table = $tables['content_index'];

		// Extract title from content if not provided
		if ( empty( $title ) ) {
			$lines = explode( "\n", $content );
			$title = ! empty( $lines[0] ) ? substr( $lines[0], 0, 255 ) : $object_type . ' #' . $object_id;
		}

		// Plain text for the AI: "&#36;80.82" → "$80.82", "&amp;" → "&".
		$content = html_entity_decode( (string) $content, ENT_QUOTES | ENT_HTML5, 'UTF-8' );
		$title   = html_entity_decode( (string) $title, ENT_QUOTES | ENT_HTML5, 'UTF-8' );

		$existing = $wpdb->get_row( $wpdb->prepare(
			"SELECT id, content, title, url, embedding_status FROM {$content_index_table} WHERE object_type = %s AND object_id = %s",
			$object_type,
			$object_id
		), ARRAY_A );

		// Unchanged content: nothing to re-index or re-embed (saves an embedding
		// API call on every product/order save that doesn't touch the text).
		if ( $existing && $existing['content'] === $content && $existing['title'] === $title && $existing['url'] === $url
			&& ( ! get_option( 'askany_rag_enabled', false ) || in_array( ( $existing['embedding_status'] ?? '' ), array( 'completed', 'permanently_failed' ), true ) ) ) {
			// Mark as seen (a full re-index removes rows it did not see).
			$wpdb->update( $content_index_table, array( 'updated_at' => current_time( 'mysql' ) ), array( 'id' => (int) $existing['id'] ), array( '%s' ), array( '%d' ) );
			return;
		}

		// Upsert keeps the row id stable. REPLACE deleted and re-inserted the row,
		// so ids were re-used by other content and old vectors pointed at the wrong text.
		$now    = current_time( 'mysql' );
		$result = $wpdb->query( $wpdb->prepare(
			"INSERT INTO {$content_index_table} (object_type, object_id, content, title, url, updated_at)
			 VALUES (%s, %s, %s, %s, %s, %s)
			 ON DUPLICATE KEY UPDATE content = VALUES(content), title = VALUES(title), url = VALUES(url), updated_at = VALUES(updated_at)",
			$object_type,
			$object_id,
			$content,
			$title,
			$url,
			$now
		) );

		if ( $result === false ) {
			return;
		}

		// The text changed: drop the old vectors (chunk count may differ).
		if ( $existing ) {
			$wpdb->delete( $wpdb->prefix . 'askany_embeddings', array( 'content_id' => (int) $existing['id'] ), array( '%d' ) );
		}

		// Generate embeddings if RAG is enabled
		$this->generate_embeddings_for_content( $object_type, $object_id, $content, $url, $title );
	}

	/**
	 * Generate embeddings for content when RAG is enabled
	 */
	private function generate_embeddings_for_content( $object_type, $object_id, $content, $url, $title ) {
		$rag_enabled = get_option( 'askany_rag_enabled', false );
		if ( ! $rag_enabled ) {
			return;
		}

		// Orders are indexed for the admin overview only; they are never sent to the
		// public chat, so they need no embeddings (and no API call at checkout).
		if ( 'woocommerce_order' === $object_type ) {
			return;
		}

		global $wpdb;
		$tables              = $this->database->get_tables();
		$content_index_table = $tables['content_index'];

		$content_id = $wpdb->get_var( $wpdb->prepare(
			"SELECT id FROM {$content_index_table} WHERE object_type = %s AND object_id = %s",
			$object_type,
			$object_id
		) );

		if ( ! $content_id ) {
			return;
		}

		$api_manager       = new ApiManager();
		$embedding_manager = new Embedding_Manager( $this->database, $api_manager );
		$excerpt           = substr( wp_strip_all_tags( $content ), 0, 500 );

		$meta = array(
			'source_url' => $url,
			'title'      => $title,
			'excerpt'    => $excerpt,
		);

		// P3.2 — When Pinecone is enabled, upsert vectors there instead of (or in
		// addition to) the local embeddings table. The excerpt travels WITH the
		// vector in Pinecone metadata so no second DB fetch is needed at query time.
		if ( Pinecone_Handler::is_enabled() ) {
			$embedding_data = $embedding_manager->generate_embedding( $content );
			if ( ! is_wp_error( $embedding_data ) ) {
				$pinecone = new Pinecone_Handler();
				$vector_id = $object_type . '_' . $content_id;
				$pinecone->upsert_vector(
					$vector_id,
					$embedding_data['embedding'],
					array(
						'content_id'   => (int) $content_id,
						'content_type' => $object_type,
						'source_url'   => $url,
						'title'        => $title,
						'excerpt'      => $excerpt,
						'chunk_index'  => 0,
						'total_chunks' => 1,
					)
				);
			}
			// Still store locally as backup / for admin views
		}

		// Always generate local embeddings (used as fallback when Pinecone disabled)
		$result = $embedding_manager->generate_content_embeddings(
			$content_id,
			$object_type,
			$content,
			$meta
		);

		if ( is_wp_error( $result ) ) {
			$error_message  = $result->get_error_message();
			$is_quota_error = stripos( $error_message, 'quota' ) !== false
			                  || stripos( $error_message, 'rate limit' ) !== false;
			if ( ! $is_quota_error ) {
				// error_log( 'AskAny: Failed to generate embeddings for ' . $object_type . ' ' . $object_id . ': ' . $error_message );
			}
		}
	}

	/**
	 * Prepare content for indexing.
	 *
	 * This method prepares the content by removing HTML tags, shortcodes,
	 * and extra whitespace.
	 *
	 * @param string $content Content to prepare.
	 * @return string Prepared content.
	 */
	private function prepare_content( $content ) {
		// Extract Elementor content if present
		$content = $this->extract_elementor_content( $content );

		// Extract Gutenberg blocks content if present
		$content = $this->extract_gutenberg_content( $content );

		// Keep block structure: a line break after paragraphs, headings, list
		// items and rows, so chunks split on real boundaries.
		$content = preg_replace( '#(</(p|div|h[1-6]|li|tr|section|article|blockquote|figcaption|dd|dt)>|<br\s*/?>)#i', "$1\n", (string) $content );

		// Remove HTML tags (line breaks are kept).
		$content = wp_strip_all_tags( $content );

		// Shortcodes: drop the tags but keep the text inside them. strip_shortcodes()
		// deleted the inner text of page builders (Divi, WPBakery, Avada...).
		$content = $this->strip_shortcode_tags( $content );

		// Collapse spaces, keep paragraph breaks.
		$content = preg_replace( '/[ \t\x{00A0}]+/u', ' ', $content );
		$content = preg_replace( '/ *\n */', "\n", (string) $content );
		$content = preg_replace( '/\n{3,}/', "\n\n", (string) $content );

		return trim( (string) $content );
	}

	/**
	 * Remove shortcode tags but keep their inner text.
	 *
	 * @param string $content Text.
	 * @return string
	 */
	private function strip_shortcode_tags( $content, $depth = 0 ) {
		$content = (string) $content;
		if ( false === strpos( $content, '[' ) ) {
			return $content;
		}
		// Registered shortcodes: keep the enclosed content (nested ones too).
		$regex   = get_shortcode_regex();
		$content = (string) preg_replace_callback(
			'/' . $regex . '/',
			function ( $m ) use ( $depth ) {
				// [[escaped]] stays literal text.
				if ( '[' === $m[1] && ']' === $m[6] ) {
					return substr( $m[0], 1, -1 );
				}
				$inner = isset( $m[5] ) ? (string) $m[5] : '';
				return ' ' . ( $depth < 5 ? $this->strip_shortcode_tags( $inner, $depth + 1 ) : $inner ) . ' ';
			},
			$content
		);
		// Page-builder tags whose plugin is not active (raw [vc_row], [et_pb_text]...).
		return (string) preg_replace( '/\[\/?(vc_|et_pb_|fusion_|av_|cs_|ux_|tatsu_|mk_|rev_slider|elementor-template)[^\]]*\]/i', ' ', $content );
	}

	/**
	 * Append extracted builder/block text, skipping lines the content already
	 * has (the post content usually repeats the same text, which doubled items).
	 *
	 * @param string $content   Original content.
	 * @param string $extracted Extracted text.
	 * @return string
	 */
	private function append_new_lines( $content, $extracted ) {
		$haystack = mb_strtolower( (string) preg_replace( '/\s+/u', ' ', wp_strip_all_tags( (string) $content ) ) );
		$extra    = array();
		foreach ( (array) preg_split( '/\R/u', (string) $extracted ) as $line ) {
			$line = trim( (string) $line );
			if ( '' === $line ) {
				continue;
			}
			$needle = mb_strtolower( (string) preg_replace( '/\s+/u', ' ', $line ) );
			if ( false === mb_strpos( $haystack, $needle ) ) {
				$extra[]   = $line;
				$haystack .= ' ' . $needle;
			}
		}
		return empty( $extra ) ? (string) $content : $content . "\n\n" . implode( "\n", $extra );
	}
	
	/**
	 * Extract content from Elementor data
	 *
	 * @param string $content Post content
	 * @return string Enhanced content with Elementor data
	 */
	private function extract_elementor_content( $content ) {
		global $post;
		
		// Check if Elementor is active and this post uses Elementor
		if ( ! class_exists( '\Elementor\Plugin' ) ) {
			return $content;
		}
		
		// Get current post ID from global or try to extract from context
		$post_id = $post ? $post->ID : 0;
		
		if ( ! $post_id ) {
			return $content;
		}
		
		// Check if this page is built with Elementor
		$elementor_data = get_post_meta( $post_id, '_elementor_data', true );
		
		if ( empty( $elementor_data ) ) {
			return $content;
		}
		
		// Decode Elementor JSON data
		$elementor_content = json_decode( $elementor_data, true );
		
		if ( ! is_array( $elementor_content ) ) {
			return $content;
		}
		
		// Extract text content from Elementor elements
		$extracted_text = $this->parse_elementor_elements( $elementor_content );
		
		// Combine original content with Elementor content (new lines only)
		if ( ! empty( $extracted_text ) ) {
			$content = $this->append_new_lines( $content, $extracted_text );
		}
		
		return $content;
	}
	
	/**
	 * Recursively parse Elementor elements to extract text
	 *
	 * @param array $elements Elementor elements array
	 * @return string Extracted text content
	 */
	private function parse_elementor_elements( $elements ) {
		$text_content = array();
		
		foreach ( $elements as $element ) {
			// Extract widget content
			if ( isset( $element['widgetType'] ) ) {
				$widget_type = $element['widgetType'];
				$settings = isset( $element['settings'] ) ? $element['settings'] : array();
				
				// Extract text from common widgets
				switch ( $widget_type ) {
					case 'heading':
						if ( ! empty( $settings['title'] ) ) {
							$text_content[] = $settings['title'];
						}
						break;
						
					case 'text-editor':
						if ( ! empty( $settings['editor'] ) ) {
							$text_content[] = wp_strip_all_tags( $settings['editor'] );
						}
						break;
						
					case 'button':
						if ( ! empty( $settings['text'] ) ) {
							$text_content[] = $settings['text'];
						}
						break;
						
					case 'image':
						if ( ! empty( $settings['caption'] ) ) {
							$text_content[] = $settings['caption'];
						}
						if ( ! empty( $settings['alt_text'] ) ) {
							$text_content[] = $settings['alt_text'];
						}
						break;
						
					case 'icon-box':
						if ( ! empty( $settings['title_text'] ) ) {
							$text_content[] = $settings['title_text'];
						}
						if ( ! empty( $settings['description_text'] ) ) {
							$text_content[] = wp_strip_all_tags( $settings['description_text'] );
						}
						break;
						
					case 'testimonial':
						if ( ! empty( $settings['testimonial_content'] ) ) {
							$text_content[] = wp_strip_all_tags( $settings['testimonial_content'] );
						}
						if ( ! empty( $settings['testimonial_name'] ) ) {
							$text_content[] = $settings['testimonial_name'];
						}
						break;
						
					case 'accordion':
					case 'toggle':
						if ( ! empty( $settings['tabs'] ) && is_array( $settings['tabs'] ) ) {
							foreach ( $settings['tabs'] as $tab ) {
								if ( ! empty( $tab['tab_title'] ) ) {
									$text_content[] = $tab['tab_title'];
								}
								if ( ! empty( $tab['tab_content'] ) ) {
									$text_content[] = wp_strip_all_tags( $tab['tab_content'] );
								}
							}
						}
						break;
						
					case 'pricing-table':
						if ( ! empty( $settings['heading'] ) ) {
							$text_content[] = $settings['heading'];
						}
						if ( ! empty( $settings['sub_heading'] ) ) {
							$text_content[] = $settings['sub_heading'];
						}
						if ( ! empty( $settings['price'] ) ) {
							$text_content[] = 'Price: ' . $settings['price'];
						}
						if ( ! empty( $settings['features_list'] ) && is_array( $settings['features_list'] ) ) {
							foreach ( $settings['features_list'] as $feature ) {
								if ( ! empty( $feature['item_text'] ) ) {
									$text_content[] = wp_strip_all_tags( $feature['item_text'] );
								}
							}
						}
						break;
				}
			}
			
			// Recursively parse child elements
			if ( ! empty( $element['elements'] ) && is_array( $element['elements'] ) ) {
				$child_content = $this->parse_elementor_elements( $element['elements'] );
				if ( ! empty( $child_content ) ) {
					$text_content[] = $child_content;
				}
			}
		}
		
		return implode( "\n", $text_content );
	}
	
	/**
	 * Extract content from Gutenberg blocks
	 *
	 * @param string $content Post content with Gutenberg blocks
	 * @return string Enhanced content with block data extracted
	 */
	private function extract_gutenberg_content( $content ) {
		// Check if content has Gutenberg blocks
		if ( ! has_blocks( $content ) ) {
			return $content;
		}
		
		// Parse blocks
		$blocks = parse_blocks( $content );
		
		if ( empty( $blocks ) ) {
			return $content;
		}
		
		// Extract text from blocks
		$extracted_text = $this->parse_gutenberg_blocks( $blocks );
		
		// Combine original content with extracted block content (new lines only)
		if ( ! empty( $extracted_text ) ) {
			$content = $this->append_new_lines( $content, $extracted_text );
		}
		
		return $content;
	}
	
	/**
	 * Recursively parse Gutenberg blocks to extract text
	 *
	 * @param array $blocks Gutenberg blocks array
	 * @return string Extracted text content
	 */
	private function parse_gutenberg_blocks( $blocks ) {
		$text_content = array();
		
		foreach ( $blocks as $block ) {
			// Skip empty blocks
			if ( empty( $block['blockName'] ) ) {
				continue;
			}
			
			$block_name = $block['blockName'];
			$attrs = isset( $block['attrs'] ) ? $block['attrs'] : array();
			
			// Extract content based on block type
			switch ( $block_name ) {
				case 'core/paragraph':
				case 'core/heading':
				case 'core/quote':
				case 'core/verse':
				case 'core/preformatted':
					if ( ! empty( $block['innerHTML'] ) ) {
						$text_content[] = wp_strip_all_tags( $block['innerHTML'] );
					}
					break;
					
				case 'core/list':
					if ( ! empty( $block['innerHTML'] ) ) {
						$text_content[] = wp_strip_all_tags( $block['innerHTML'] );
					}
					break;
					
				case 'core/image':
				case 'core/cover':
					if ( ! empty( $attrs['alt'] ) ) {
						$text_content[] = $attrs['alt'];
					}
					if ( ! empty( $attrs['caption'] ) ) {
						$text_content[] = $attrs['caption'];
					}
					break;
					
				case 'core/button':
					if ( ! empty( $block['innerHTML'] ) ) {
						$text_content[] = wp_strip_all_tags( $block['innerHTML'] );
					}
					break;
					
				case 'core/columns':
				case 'core/column':
				case 'core/group':
					// These are container blocks, process inner blocks
					break;
					
				case 'core/table':
					if ( ! empty( $block['innerHTML'] ) ) {
						$text_content[] = wp_strip_all_tags( $block['innerHTML'] );
					}
					break;
			}
			
			// Recursively parse inner blocks
			if ( ! empty( $block['innerBlocks'] ) && is_array( $block['innerBlocks'] ) ) {
				$inner_content = $this->parse_gutenberg_blocks( $block['innerBlocks'] );
				if ( ! empty( $inner_content ) ) {
					$text_content[] = $inner_content;
				}
			}
		}
		
		return implode( "\n", $text_content );
	}
	
	/**
	 * Index WooCommerce store configuration.
	 *
	 * This method indexes store settings like shipping methods, payment gateways,
	 * and store policies to help AI answer customer support questions.
	 *
	 * @return void
	 */
	private function index_store_configuration() {
		if ( ! class_exists('WooCommerce') ) {
			return;
		}
		
		$content = array();
		
		// Index shipping zones and methods
		if ( class_exists('WC_Shipping_Zones') ) {
			$shipping_zones = \WC_Shipping_Zones::get_zones();
			if ( ! empty($shipping_zones) ) {
				$content[] = "=== SHIPPING INFORMATION ===";
				foreach ( $shipping_zones as $zone ) {
					$zone_obj = new \WC_Shipping_Zone($zone['id']);
					$content[] = "Shipping Zone: " . $zone_obj->get_zone_name();
					$content[] = "Zone Locations: " . implode(', ', array_column($zone['zone_locations'], 'code'));
					
					$shipping_methods = $zone_obj->get_shipping_methods(true);
					foreach ( $shipping_methods as $method ) {
						$content[] = "Shipping Method: " . $method->get_title();
						if ( $method->get_method_description() ) {
							$content[] = "Description: " . $method->get_method_description();
						}
					}
				}
				
				// Worldwide shipping (zone 0)
				$zone_0 = new \WC_Shipping_Zone(0);
				$methods_0 = $zone_0->get_shipping_methods(true);
				if ( ! empty($methods_0) ) {
					$content[] = "Worldwide Shipping Methods:";
					foreach ( $methods_0 as $method ) {
						$content[] = "- " . $method->get_title();
					}
				}
			}
		}
		
		// Index payment gateways (names only, no credentials)
		$payment_gateways = WC()->payment_gateways->get_available_payment_gateways();
		if ( ! empty($payment_gateways) ) {
			$content[] = "\n=== PAYMENT METHODS ===";
			foreach ( $payment_gateways as $gateway ) {
				$content[] = "Payment Method: " . $gateway->get_title();
				if ( $gateway->get_description() ) {
					$content[] = "Description: " . $gateway->get_description();
				}
			}
		}
		
		// Index store policies from WooCommerce settings
		$policy_pages = array(
			'woocommerce_terms_page_id' => 'Terms and Conditions',
			'woocommerce_privacy_page_id' => 'Privacy Policy',
			'wp_page_for_privacy_policy' => 'Privacy Policy',
		);
		
		$content[] = "\n=== STORE POLICIES ===";
		foreach ( $policy_pages as $option_key => $label ) {
			$page_id = get_option($option_key);
			if ( $page_id ) {
				$page = get_post($page_id);
				if ( $page && $page->post_status === 'publish' ) {
					$content[] = $label . ": " . $page->post_title;
					$content[] = strip_tags($page->post_content);
				}
			}
		}
		
		// Index common store pages
		$store_pages = array(
			'woocommerce_shop_page_id' => 'Shop Page',
			'woocommerce_cart_page_id' => 'Cart Page',
			'woocommerce_checkout_page_id' => 'Checkout Page',
			'woocommerce_myaccount_page_id' => 'My Account Page',
		);
		
		$content[] = "\n=== STORE PAGES ===";
		foreach ( $store_pages as $option_key => $label ) {
			$page_id = get_option($option_key);
			if ( $page_id ) {
				$page = get_post($page_id);
				if ( $page ) {
					$content[] = $label . ": " . get_permalink($page_id);
				}
			}
		}
		
		// Index currency and store location
		$content[] = "\n=== STORE SETTINGS ===";
		$content[] = "Currency: " . get_woocommerce_currency();
		$content[] = "Currency Symbol: " . get_woocommerce_currency_symbol();
		$content[] = "Store Country: " . WC()->countries->get_base_country();
		$content[] = "Store Address: " . WC()->countries->get_base_address();
		$content[] = "Store City: " . WC()->countries->get_base_city();
		$content[] = "Store Postcode: " . WC()->countries->get_base_postcode();
		
		// Index tax settings
		$content[] = "\n=== TAX SETTINGS ===";
		$content[] = "Prices include tax: " . (wc_prices_include_tax() ? 'Yes' : 'No');
		$content[] = "Calculate tax based on: " . get_option('woocommerce_tax_based_on');
		$content[] = "Display prices in shop: " . get_option('woocommerce_tax_display_shop');
		$content[] = "Display prices in cart: " . get_option('woocommerce_tax_display_cart');
		
		// Index order statuses
		$order_statuses = wc_get_order_statuses();
		$content[] = "\n=== ORDER STATUSES ===";
		foreach ( $order_statuses as $status_key => $status_label ) {
			$content[] = "Order Status: " . $status_label . " (" . $status_key . ")";
		}
		
		$prepared_content = $this->prepare_content(implode("\n", $content));
		if ( ! empty($prepared_content) ) {
			$this->store_content('store_config', 1, $prepared_content, get_home_url(), 'Store Configuration');
		}
	}
	
	/**
	 * Index FAQ data for AI understanding.
	 *
	 * This method indexes all FAQ questions and answers so the AI can understand
	 * and match user queries with appropriate FAQ responses intelligently.
	 *
	 * @return void
	 */
	private function index_faqs() {
		global $wpdb;
		$tables = $this->database->get_tables();
		
		// Index both home FAQs and Q&A pairs
		$this->index_home_faqs();
		$this->index_qa_pairs();
	}
	
	/**
	 * Index home page FAQs.
	 *
	 * @return void
	 */
	private function index_home_faqs() {
		global $wpdb;
		$tables = $this->database->get_tables();
		$faqs_table = $tables['home_faqs'];
		
		// Check if table exists first
		if ( $wpdb->get_var("SHOW TABLES LIKE '$faqs_table'") !== $faqs_table ) {
			// error_log("AskAny: Home FAQ table does not exist: $faqs_table");
			return;
		}
		
		// Only active FAQs: a FAQ the admin switched off must not answer in the chat.
		$faqs = $wpdb->get_results("SELECT * FROM $faqs_table WHERE is_active = 1 ORDER BY id ASC");

		// Remove index rows of FAQs that are now inactive or deleted.
		$active_ids = array_map( 'intval', wp_list_pluck( (array) $faqs, 'id' ) );
		$indexed    = $wpdb->get_col( "SELECT object_id FROM {$tables['content_index']} WHERE object_type = 'faq'" );
		foreach ( (array) $indexed as $faq_object_id ) {
			if ( ! in_array( (int) $faq_object_id, $active_ids, true ) ) {
				$this->remove_index_row( 'faq', (int) $faq_object_id );
			}
		}
		
		if ( empty($faqs) ) {
			// error_log("AskAny: No home FAQs found in table $faqs_table");
			return;
		}
		
		
		$content = array();
		$content[] = "=== HOME PAGE FREQUENTLY ASKED QUESTIONS ===";
		$content[] = "This section contains home page FAQ questions and answers for intelligent matching.";
		$content[] = "";
		
		foreach ( $faqs as $faq ) {
			$content[] = "--- FAQ ITEM ---";
			$content[] = "Question: " . $faq->question;
			$content[] = "Answer: " . $faq->answer;
			$content[] = "Category: Home FAQ";
			$content[] = "Type: Frequently Asked Question";
			$content[] = "";
			
			// Also index each FAQ individually for better matching
			$individual_content = array(
				"=== FAQ ITEM ===",
				"Question: " . $faq->question,
				"Answer: " . $faq->answer,
				"FAQ Category: Home Page",
				"Content Type: FAQ",
			);
			
			$prepared_individual = $this->prepare_content(implode("\n", $individual_content));
			if ( ! empty($prepared_individual) ) {
				$this->store_content('faq', $faq->id, $prepared_individual, '', 'FAQ: ' . $faq->question);
			} 
			
		}
		
		// NOTE: faq_collection removed — individual faq items are sufficient for RAG retrieval.
		// A collection document blobs the entire FAQ database into one embedding, causing
		// the whole FAQ set to be returned as context on any FAQ-related query.
	}
	
	/**
	 * Index Q&A pairs from Q&A Management.
	 *
	 * @return void
	 */
	private function index_qa_pairs() {
		global $wpdb;
		$tables = $this->database->get_tables();
		$qa_table = $tables['qa_pairs'];
		
		// Check if table exists first
		if ( $wpdb->get_var("SHOW TABLES LIKE '$qa_table'") !== $qa_table ) {
			// error_log("AskAny: Q&A pairs table does not exist: $qa_table");
			return;
		}
		
		// Get all active Q&A pairs from database
		$qa_pairs = $wpdb->get_results("SELECT * FROM $qa_table WHERE status = 'active' ORDER BY id ASC");
		
	
		if ( empty($qa_pairs) ) {
			// error_log("AskAny: No Q&A pairs found in table $qa_table");
			return;
		}
		
		
		$content = array();
		$content[] = "=== Q&A KNOWLEDGE BASE ===";
		$content[] = "This section contains Q&A pairs from the knowledge base for intelligent matching.";
		$content[] = "";
		
		foreach ( $qa_pairs as $qa ) {
			$content[] = "--- Q&A ITEM ---";
			$content[] = "Question: " . $qa->question;
			$content[] = "Answer: " . $qa->answer;
			$content[] = "Category: Q&A Knowledge Base";
			$content[] = "Type: Question and Answer";
			$content[] = "";
			
			// Also index each Q&A individually for better matching
			$individual_content = array(
				"=== Q&A ITEM ===",
				"Question: " . $qa->question,
				"Answer: " . $qa->answer,
				"Q&A Category: Knowledge Base",
				"Content Type: Q&A",
			);
			
			$prepared_individual = $this->prepare_content(implode("\n", $individual_content));
			if ( ! empty($prepared_individual) ) {
				// Use 'qa' prefix to differentiate from 'faq'
				$this->store_content('qa', $qa->id, $prepared_individual, '', 'Q&A: ' . $qa->question);
			} 
		}
		
		// NOTE: qa_collection removed — individual qa items are sufficient for RAG retrieval.
		// A collection document blobs the entire Q&A database into one embedding.
	}
	
	/**
	 * Index PDF content for AI understanding.
	 *
	 * This method indexes all uploaded PDF content so the AI can reference
	 * document information when answering user queries.
	 *
	 * @return void
	 */
	private function index_pdfs() {
		global $wpdb;
		$tables = $this->database->get_tables();
		$pdf_table = $tables['pdf_index'];
		
		// Check if table exists first
		if ( $wpdb->get_var("SHOW TABLES LIKE '$pdf_table'") !== $pdf_table ) {
			// error_log("AskAny: PDF table does not exist: $pdf_table");
			return;
		}
		
		// Get all PDF content from database
		$pdfs = $wpdb->get_results("SELECT * FROM $pdf_table ORDER BY id ASC");
		
		
		if ( empty($pdfs) ) {
			// error_log("AskAny: No PDFs found in table $pdf_table");
			return;
		}
		
		
		$content = array();
		$content[] = "=== PDF DOCUMENTS ===";
		$content[] = "This section contains content from uploaded PDF documents.";
		$content[] = "";
		
		foreach ( $pdfs as $pdf ) {
			$pdf_title = isset($pdf->title) ? $pdf->title : 'PDF Document';
			$pdf_filename = isset($pdf->file_name) ? $pdf->file_name : 'Unknown';
			
			$content[] = "--- PDF DOCUMENT ---";
			$content[] = "Document Title: " . $pdf_title;
			$content[] = "File Name: " . $pdf_filename;
			$content[] = "Content: " . $pdf->content;
			$content[] = "Document Type: PDF";
			$content[] = "Source: Uploaded Document";
			$content[] = "";
			
			// Also index each PDF individually
			$individual_content = array(
				"=== PDF DOCUMENT ===",
				"Document Title: " . $pdf_title,
				"File Name: " . $pdf_filename,
				"Content: " . $pdf->content,
				"Document Type: PDF Document",
				"Source: Uploaded PDF",
			);
			
			$prepared_individual = $this->prepare_content(implode("\n", $individual_content));
			if ( ! empty($prepared_individual) ) {
				$title = $pdf_title !== 'PDF Document' ? $pdf_title : ('PDF: ' . $pdf_filename);
				$this->store_content('pdf', $pdf->id, $prepared_individual, '', $title);
				
			} 
		}
		
		// NOTE: pdf_collection removed — individual pdf items are sufficient for RAG retrieval.
	}
	
	/**
	 * Index external sources for AI understanding.
	 *
	 * Reads from content_index (object_type = 'external_url') where the crawler
	 * already stores the actual page content. The external_sources tracking table
	 * only holds crawl metadata (url, status, follow_links) — no content.
	 * This function ensures embeddings exist for any external_url rows that are missing them.
	 *
	 * @return void
	 */
	private function index_external_sources() {
		global $wpdb;
		$tables             = $this->database->get_tables();
		$content_index_table = $tables['content_index'];

		// Check table exists
		if ( $wpdb->get_var( "SHOW TABLES LIKE '{$content_index_table}'" ) !== $content_index_table ) {
			return;
		}

		// Legacy rows (object_type 'external_source') held crawl progress JSON, not
		// page text: remove them so they never reach the AI.
		$legacy_ids = $wpdb->get_col( "SELECT id FROM {$content_index_table} WHERE object_type = 'external_source'" );
		if ( ! empty( $legacy_ids ) ) {
			$legacy_ids = implode( ',', array_map( 'intval', $legacy_ids ) );
			$wpdb->query( "DELETE FROM {$wpdb->prefix}askany_embeddings WHERE content_id IN ({$legacy_ids})" );
			$wpdb->query( "DELETE FROM {$content_index_table} WHERE id IN ({$legacy_ids})" );
		}

		if ( ! get_option( 'askany_rag_enabled', false ) ) {
			return; // keyword search reads the page text directly
		}

		// Crawled pages without (complete) embeddings yet.
		$rows = $wpdb->get_results(
			"SELECT id AS content_id, object_id, content, title, url
			 FROM {$content_index_table}
			 WHERE object_type = 'external_url'
			   AND ( embedding_status IS NULL OR embedding_status NOT IN ('completed','permanently_failed') )
			 ORDER BY id ASC
			 LIMIT 100"
		);

		if ( empty( $rows ) ) {
			return;
		}

		foreach ( $rows as $row ) {
			// generate_embeddings_for_content re-checks RAG enabled flag internally
			$this->generate_embeddings_for_content(
				'external_url',
				$row->object_id,
				$row->content,
				$row->url,
				$row->title
			);
		}
	}
	
	/**
	 * Get indexed content by type for viewing.
	 *
	 * This method retrieves indexed content by object type for admin viewing.
	 *
	 * @param string $object_type The content type to retrieve.
	 * @param int $limit Optional limit for results.
	 * @param int $offset Optional offset for pagination.
	 * @return array Array of indexed content items.
	 */
	public function get_indexed_content_by_type( $object_type, $limit = 50, $offset = 0 ) {
		global $wpdb;
		if ( 'external_source' === $object_type ) {
			$object_type = 'external_url'; // crawled pages are stored per URL
		}
		$tables = $this->database->get_tables();
		$content_index_table = $tables['content_index'];
		
		if ( $object_type === 'custom_db' ) {
			$results = $wpdb->get_results($wpdb->prepare(
				"SELECT id, object_id, object_type, title, url, 
				LEFT(content, 200) as content_preview, 
				LENGTH(content) as content_length,
				created_at, updated_at 
				FROM $content_index_table 
				WHERE object_type LIKE %s 
				ORDER BY updated_at DESC 
				LIMIT %d OFFSET %d",
				'custom_db_%',
				$limit,
				$offset
			));
		} else {
			$results = $wpdb->get_results($wpdb->prepare(
				"SELECT id, object_id, object_type, title, url, 
				LEFT(content, 200) as content_preview, 
				LENGTH(content) as content_length,
				created_at, updated_at 
				FROM $content_index_table 
				WHERE object_type = %s 
				ORDER BY updated_at DESC 
				LIMIT %d OFFSET %d",
				$object_type,
				$limit,
				$offset
			));
		}
		
		return $results;
	}
	
	/**
	 * Get full indexed content item by ID.
	 *
	 * This method retrieves a complete indexed content item by its ID.
	 *
	 * @param int $content_id The content index ID.
	 * @return object|null The content item or null if not found.
	 */
	public function get_indexed_content_by_id( $content_id ) {
		global $wpdb;
		$tables = $this->database->get_tables();
		$content_index_table = $tables['content_index'];
		
		$result = $wpdb->get_row($wpdb->prepare(
			"SELECT * FROM $content_index_table WHERE id = %d",
			$content_id
		));
		
		return $result;
	}
	
	/**
	 * Delete a specific object from content index and RAG vector store.
	 *
	 * @param string $object_type Object type slug.
	 * @param mixed  $object_id   Object ID or slug.
	 * @return void
	 */
	public function delete_object_index( $object_type, $object_id ) {
		global $wpdb;
		$tables              = $this->database->get_tables();
		$content_index_table = $tables['content_index'];

		// Get the content_index id before deleting (needed for Pinecone vector ID)
		$content_id = $wpdb->get_var(
			$wpdb->prepare(
				"SELECT id FROM {$content_index_table} WHERE object_type = %s AND object_id = %s",
				$object_type,
				$object_id
			)
		);

		$wpdb->delete(
			$content_index_table,
			array( 'object_type' => $object_type, 'object_id' => $object_id ),
			array( '%s', '%s' )
		);

		// Also delete from local embeddings table
		$rag_table = $wpdb->prefix . 'askany_rag_embeddings';
		if ( $wpdb->get_var( "SHOW TABLES LIKE '$rag_table'" ) === $rag_table ) {
			$wpdb->delete(
				$rag_table,
				array( 'object_type' => $object_type, 'object_id' => $object_id ),
				array( '%s', '%s' )
			);
		}

		// P3.5 — Delete from local embeddings table
		$embeddings_table = $wpdb->prefix . 'askany_embeddings';
		if ( $content_id && $wpdb->get_var( "SHOW TABLES LIKE '{$embeddings_table}'" ) === $embeddings_table ) {
			$wpdb->delete(
				$embeddings_table,
				array( 'content_id' => (int) $content_id ),
				array( '%d' )
			);
		}

		// P3.5 — Delete from Pinecone when enabled
		if ( $content_id && \Askany\Pinecone_Handler::is_enabled() ) {
			$pinecone = new \Askany\Pinecone_Handler();
			$pinecone->delete_by_content_id( (int) $content_id, $object_type );
		}
	}

	/**
	 * Index a single Q&A Pair immediately.
	 */
	public function index_single_qa_pair( $qa_id ) {
		global $wpdb;
		$tables   = $this->database->get_tables();
		$qa_table = $tables['qa_pairs'];

		$qa = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM $qa_table WHERE id = %d", $qa_id ) );
		if ( ! $qa || $qa->status !== 'active' ) {
			$this->delete_qa_pair_index( $qa_id );
			return;
		}

		$individual_content = array(
			"Question: " . $qa->question,
			"Answer: " . $qa->answer,
			"Category: Q&A",
		);
		$prepared_individual = $this->prepare_content( implode( "\n", $individual_content ) );
		if ( ! empty( $prepared_individual ) ) {
			$this->store_content( 'qa', $qa->id, $prepared_individual, '', 'Q&A: ' . $qa->question );
		}

		// Rebuild Q&A collection summary
		$this->index_qa_pairs();
	}

	/**
	 * Delete a single Q&A Pair index entry.
	 */
	public function delete_qa_pair_index( $qa_id ) {
		$this->delete_object_index( 'qa', $qa_id );
		$this->index_qa_pairs();
	}

	/**
	 * Index a single FAQ item immediately.
	 */
	public function index_single_faq( $faq_id ) {
		global $wpdb;
		$tables    = $this->database->get_tables();
		$faq_table = $tables['home_faqs'];

		$faq = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM $faq_table WHERE id = %d", $faq_id ) );
		if ( ! $faq || empty( $faq->is_active ) ) {
			$this->delete_faq_index( $faq_id );
			return;
		}

		// Same text format as index_home_faqs(), so a later full re-index sees no change.
		$individual_content = array(
			"=== FAQ ITEM ===",
			"Question: " . $faq->question,
			"Answer: " . $faq->answer,
			"FAQ Category: Home Page",
			"Content Type: FAQ",
		);
		$prepared_individual = $this->prepare_content( implode( "\n", $individual_content ) );
		if ( ! empty( $prepared_individual ) ) {
			$this->store_content( 'faq', $faq->id, $prepared_individual, '', 'FAQ: ' . $faq->question );
		}
	}

	/**
	 * Delete a single FAQ index entry.
	 */
	public function delete_faq_index( $faq_id ) {
		$this->remove_index_row( 'faq', (int) $faq_id );
	}

	/**
	 * Index a single PDF document immediately.
	 */
	public function index_single_pdf( $pdf_id ) {
		global $wpdb;
		$tables    = $this->database->get_tables();
		$pdf_table = $tables['pdf_index'];

		$pdf = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM $pdf_table WHERE id = %d", $pdf_id ) );
		if ( ! $pdf ) {
			$this->delete_pdf_index( $pdf_id );
			return;
		}

		// Same text format as index_pdfs(), so a later full re-index sees no change.
		$pdf_title          = isset( $pdf->title ) ? $pdf->title : 'PDF Document';
		$pdf_filename       = isset( $pdf->file_name ) ? $pdf->file_name : 'Unknown';
		$individual_content = array(
			"=== PDF DOCUMENT ===",
			"Document Title: " . $pdf_title,
			"File Name: " . $pdf_filename,
			"Content: " . $pdf->content,
			"Document Type: PDF Document",
			"Source: Uploaded PDF",
		);
		$prepared_individual = $this->prepare_content( implode( "\n", $individual_content ) );
		if ( ! empty( $prepared_individual ) ) {
			$title = 'PDF Document' !== $pdf_title ? $pdf_title : ( 'PDF: ' . $pdf_filename );
			$this->store_content( 'pdf', $pdf->id, $prepared_individual, '', $title );
		}
	}

	/**
	 * Delete a single PDF document index entry.
	 */
	public function delete_pdf_index( $pdf_id ) {
		$this->remove_index_row( 'pdf', (int) $pdf_id );
	}

	/**
	 * Index a single external source immediately.
	 */
	public function index_single_external_source( $source_id ) {
		global $wpdb;
		$source_table = $wpdb->prefix . 'askany_external_sources';

		$source = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM $source_table WHERE id = %d", $source_id ) );
		if ( ! $source || $source->status !== 'completed' ) {
			return;
		}

		// The crawler stored each page's text (object_type 'external_url');
		// create the embeddings for pages that do not have them yet.
		$this->index_external_sources();
	}

	/**
	 * Delete a single external source index entry.
	 * (The source handler removes the crawled pages and their vectors.)
	 */
	public function delete_external_source_index( $source_id ) {
		$this->delete_object_index( 'external_source', $source_id );
	}

	/**
	 * Index WooCommerce Orders.
	 */
	public function index_orders() {
		if ( ! class_exists( 'WooCommerce' ) ) {
			return;
		}

		if ( function_exists( 'wc_get_orders' ) ) {
			$orders = wc_get_orders( array(
				// Newest orders only (same cap as the batch indexer); the chat looks
				// orders up live, the index only feeds the admin overview.
				'limit'   => max( 1, (int) apply_filters( 'askany_index_orders_limit', 200 ) ),
				'orderby' => 'date',
				'order'   => 'DESC',
				'return'  => 'ids',
			) );

			foreach ( $orders as $order_id ) {
				$this->index_order( $order_id );
			}
		}
	}

	/**
	 * Index a single WooCommerce order.
	 *
	 * @param int $order_id Order ID.
	 */
	public function index_order( $order_id ) {
		if ( ! class_exists( 'WooCommerce' ) ) {
			return;
		}

		$order = wc_get_order( $order_id );
		if ( ! $order ) {
			$this->delete_order_index( $order_id );
			return;
		}

		$plugin     = \Askany\Plugin::get_instance();
		$wc_handler = $plugin ? $plugin->get_woocommerce_handler() : null;

		if ( $wc_handler && method_exists( $wc_handler, 'format_order_info' ) ) {
			$formatted_info = $wc_handler->format_order_info( $order );
		} else {
			$order_number   = $order->get_order_number();
			$status         = wc_get_order_status_name( $order->get_status() );
			$total          = $order->get_formatted_order_total();
			$email          = $order->get_billing_email();
			$formatted_info = "Order #{$order_number}\nStatus: {$status}\nCustomer Email: {$email}\nTotal: {$total}";
		}

		$order_number = $order->get_order_number();
		$email        = $order->get_billing_email();
		$title        = sprintf( 'WooCommerce Order #%s (%s)', $order_number, $email );
		$url          = method_exists( $order, 'get_edit_order_url' ) ? $order->get_edit_order_url() : home_url( '/' );

		$content  = "=== WooCommerce Order Metadata ===\n";
		$content .= "Order ID: {$order_id}\n";
		$content .= "Order Number: #{$order_number}\n";
		$content .= "Billing Email: {$email}\n";
		$content .= "Customer Name: " . $order->get_formatted_billing_full_name() . "\n";
		$content .= "Date Created: " . ( $order->get_date_created() ? $order->get_date_created()->date( 'Y-m-d H:i:s' ) : '' ) . "\n\n";
		$content .= $formatted_info;

		$prepared = $this->prepare_content( $content );
		if ( ! empty( $prepared ) ) {
			$this->store_content( 'woocommerce_order', $order_id, $prepared, $url, $title );
		}
	}

	/**
	 * Delete a single WooCommerce order index entry.
	 *
	 * @param int $order_id Order ID.
	 */
	public function delete_order_index( $order_id ) {
		$this->delete_object_index( 'woocommerce_order', $order_id );
	}

	/**
	 * Get content type display names for admin interface.
	 *
	 * @return array Array of content type display names.
	 */
	public function get_content_type_labels() {
		return array(
			'post' => 'Blog Posts',
			'page' => 'Pages',
			'template_page' => 'Template Pages',
			'docs' => 'Documentation',
			'comment' => 'Comments',
			'product' => 'Products',
			'product_category' => 'Product Categories',
			'product_tag' => 'Product Tags',
			'product_attribute' => 'Product Attributes',
			'woocommerce_order' => 'WooCommerce Orders',
			'custom_field' => 'Custom Fields',
			'menu' => 'Menus',
			'widget' => 'Widgets',
			'store_config' => 'Store Configuration',
			'faq' => 'Home FAQs',
			'faq_collection' => 'FAQ Collections',
			'qa' => 'Q&A Pairs',
			'qa_collection' => 'Q&A Collections',
			'pdf' => 'PDF Documents',
			'pdf_collection' => 'PDF Collections',
			'external_source' => 'External Sources',
			'external_url' => 'External Sources',
			'external_collection' => 'External Collections',
			'sitemap_url' => 'Sitemaps',
			'custom_db' => 'Custom DB Tables',
		);
	}

	/**
	 * Index custom database tables configured in plugin settings.
	 *
	 * @return bool True on success, false on failure.
	 */
	public function index_custom_tables() {
		$custom_tables = get_option( 'askany_custom_db_tables', array() );

		if ( is_string( $custom_tables ) ) {
			$custom_tables = array_map( 'trim', explode( ',', $custom_tables ) );
		}

		if ( empty( $custom_tables ) || ! is_array( $custom_tables ) ) {
			return false;
		}

		$options = array(
			'create_chunks' => true,
			'chunk_size'    => 10,
			'limit'         => 1000,
		);

		foreach ( $custom_tables as $table_name ) {
			if ( ! empty( trim( $table_name ) ) ) {
				$this->database->index_custom_table_data( trim( $table_name ), $options );
			}
		}

		return true;
	}
}