<?php
/**
 * External Sources Handler class for managing external URL content.
 *
 * This class handles the crawling, processing, and management of external website content
 * for the Ask Any plugin, similar to the External Sources feature.
 *
 * @package Askany
 * @since 1.3.1
 */

namespace Askany;

/**
 * External Sources Handler class
 *
 * @package Askany
 * @since 1.3.1
 */
class ExternalSourcesHandler {
	/**
	 * Database instance for managing external sources data.
	 *
	 * @var Database
	 */
	private $database;

	/**
	 * External sources table name.
	 *
	 * @var string
	 */
	private $external_sources_table;

	/**
	 * Maximum crawl depth.
	 *
	 * @var int
	 */
	private $max_depth = 2;

	/**
	 * Maximum pages to crawl per source.
	 *
	 * @var int
	 */
	private $max_pages = 30;

	/**
	 * Request timeout in seconds.
	 *
	 * @var int
	 */
	private $request_timeout = 15;

	/**
	 * Processing batch size.
	 *
	 * @var int
	 */
	private $batch_size = 5;

	/**
	 * Constructor for ExternalSourcesHandler.
	 *
	 * @param Database $database Database instance.
	 */
	public function __construct( $database ) {
		$this->database = $database;
		global $wpdb;
		$this->external_sources_table = $wpdb->prefix . 'askany_external_sources';
	}

	/**
	 * Initialize the External Sources Handler.
	 *
	 * @return void
	 */
	public function init() {
		// Create external sources table
		$this->create_external_sources_table();
		
		// Run database migrations
		$this->run_database_migrations();

		// Register REST API endpoints
		add_action( 'rest_api_init', array( $this, 'register_rest_routes' ) );

		// Register background processing action
		add_action( 'askany_process_external_url', array( $this, 'process_external_url_background' ), 10, 2 );

		// Re-crawl sources regularly so new or changed pages reach the AI.
		add_action( 'askany_refresh_external_sources', array( $this, 'refresh_stale_sources' ) );
		if ( ! wp_next_scheduled( 'askany_refresh_external_sources' ) ) {
			wp_schedule_event( time() + HOUR_IN_SECONDS, 'daily', 'askany_refresh_external_sources' );
		}
	}

	/**
	 * Re-crawl completed sources older than N days (default 7, filter
	 * askany_external_refresh_days; 0 turns it off). At most 3 per run.
	 *
	 * @return void
	 */
	public function refresh_stale_sources() {
		global $wpdb;
		$days = (int) apply_filters( 'askany_external_refresh_days', 7 );
		if ( $days <= 0 ) {
			return;
		}
		$cutoff = gmdate( 'Y-m-d H:i:s', current_time( 'timestamp' ) - $days * DAY_IN_SECONDS ); // phpcs:ignore WordPress.DateTime.CurrentTimeTimestamp.Requested
		$sources = $wpdb->get_results( $wpdb->prepare(
			"SELECT id, url FROM {$this->external_sources_table} WHERE status IN ('completed','failed') AND ( last_crawled IS NULL OR last_crawled < %s ) ORDER BY last_crawled ASC LIMIT 3",
			$cutoff
		) );
		foreach ( (array) $sources as $i => $source ) {
			wp_schedule_single_event( time() + 60 + $i * 300, 'askany_process_external_url', array( (int) $source->id, $source->url ) );
		}
	}

	/**
	 * Create the external sources table.
	 *
	 * @return void
	 */
	private function create_external_sources_table() {
		global $wpdb;
		require_once ABSPATH . 'wp-admin/includes/upgrade.php';

		$charset_collate = $wpdb->get_charset_collate();

		$sql = "CREATE TABLE IF NOT EXISTS {$this->external_sources_table} (
			id bigint(20) NOT NULL AUTO_INCREMENT,
			url varchar(500) NOT NULL,
			title varchar(255) DEFAULT '',
			status varchar(20) NOT NULL DEFAULT 'pending',
			follow_links tinyint(1) NOT NULL DEFAULT 0,
			include_paths text NULL,
			exclude_paths text NULL,
			include_selectors text NULL,
			exclude_selectors text NULL,
			crawled_urls longtext NULL,
			processing_progress text NULL,
			last_crawled datetime NULL,
			error_message text NULL,
			created_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
			updated_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
			PRIMARY KEY (id),
			KEY url (url(191)),
			KEY status (status),
			KEY last_crawled (last_crawled)
		) $charset_collate;";

		dbDelta( $sql );
	}
	
	/**
	 * Run database migrations for external sources table.
	 * 
	 * This method checks if migrations are needed and applies them.
	 * Uses WordPress options to track which migrations have been run.
	 *
	 * @return void
	 */
	private function run_database_migrations() {
		global $wpdb;
		
		// Get current migration version
		$current_version = get_option( 'askany_external_sources_db_version', '1.0.0' );
		$target_version = '1.1.0'; // Version with processing_progress column
		
		// Check if migration is needed
		if ( version_compare( $current_version, $target_version, '<' ) ) {
			// Check if processing_progress column exists
			$column_exists = $wpdb->get_results( 
				$wpdb->prepare(
					"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS 
					WHERE TABLE_SCHEMA = %s 
					AND TABLE_NAME = %s 
					AND COLUMN_NAME = 'processing_progress'",
					DB_NAME,
					$this->external_sources_table
				)
			);
			
			// Add processing_progress column if it doesn't exist
			if ( empty( $column_exists ) ) {
				$wpdb->query(
					"ALTER TABLE {$this->external_sources_table} 
					ADD COLUMN processing_progress text NULL 
					AFTER crawled_urls"
				);
				
				// Log the migration
				// error_log( 'AskAny: Added processing_progress column to external sources table' );
			}
			
			// Update migration version
			update_option( 'askany_external_sources_db_version', $target_version );
		}
	}

	/**
	 * Register REST API routes for external sources.
	 *
	 * @return void
	 */
	public function register_rest_routes() {
		// Add external source
		register_rest_route( 'askany/v1', '/external-sources', array(
			'methods' => 'POST',
			'callback' => array( $this, 'add_external_source' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			},
			'args' => array(
				'url' => array(
					'required' => true,
					'type' => 'string',
					'validate_callback' => function( $param ) {
						return filter_var( $param, FILTER_VALIDATE_URL );
					}
				),
				'follow_links' => array(
					'required' => false,
					'type' => 'boolean',
					'default' => false
				),
				'include_paths' => array(
					'required' => false,
					'type' => 'string',
					'default' => ''
				),
				'exclude_paths' => array(
					'required' => false,
					'type' => 'string',
					'default' => ''
				),
				'include_selectors' => array(
					'required' => false,
					'type' => 'string',
					'default' => ''
				),
				'exclude_selectors' => array(
					'required' => false,
					'type' => 'string',
					'default' => ''
				)
			)
		) );

		// Get external sources
		register_rest_route( 'askany/v1', '/external-sources', array(
			'methods' => 'GET',
			'callback' => array( $this, 'get_external_sources' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );

		// Delete external source
		register_rest_route( 'askany/v1', '/external-sources/(?P<id>\d+)', array(
			'methods' => 'DELETE',
			'callback' => array( $this, 'delete_external_source' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );

		// Get crawled URLs for a source
		register_rest_route( 'askany/v1', '/external-sources/(?P<id>\d+)/urls', array(
			'methods' => 'GET',
			'callback' => array( $this, 'get_crawled_urls' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );

		// Reprocess external source
		register_rest_route( 'askany/v1', '/external-sources/(?P<id>\d+)/reprocess', array(
			'methods' => 'POST',
			'callback' => array( $this, 'reprocess_external_source' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );

		// Check processing status
		register_rest_route( 'askany/v1', '/external-sources/status', array(
			'methods' => 'GET',
			'callback' => array( $this, 'check_processing_status' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );
	}

	/**
	 * Add a new external source.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response|\WP_Error The response object or error.
	 */
	public function add_external_source( $request ) {
		global $wpdb;

		$url = esc_url_raw( $request->get_param( 'url' ) );
		$follow_links = (bool) $request->get_param( 'follow_links' );
		$include_paths = sanitize_text_field( $request->get_param( 'include_paths' ) );
		$exclude_paths = sanitize_text_field( $request->get_param( 'exclude_paths' ) );
		$include_selectors = sanitize_text_field( $request->get_param( 'include_selectors' ) );
		$exclude_selectors = sanitize_text_field( $request->get_param( 'exclude_selectors' ) );


		if ( empty( $url ) || ! wp_http_validate_url( $url ) ) {
			return new \WP_Error( 'invalid_url', __( 'Please enter a valid http(s) URL.', 'askany' ), array( 'status' => 400 ) );
		}

		// Free plan: 2 external sources (same limit the settings page shows).
		if ( ! \Askany\Plugin::get_instance()->is_pro_active() ) {
			$count = (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table}" ); // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
			if ( $count >= 2 ) {
				return new \WP_Error( 'askany_pro_required', __( 'The free version supports 2 external sources. Upgrade to AskAny Pro for more.', 'askany' ), array( 'status' => 403 ) );
			}
		}

		// Check if URL already exists
		$existing = $wpdb->get_var( $wpdb->prepare(
			"SELECT id FROM {$this->external_sources_table} WHERE url = %s",
			$url
		) );

		if ( $existing ) {
			return new \WP_Error( 'url_exists', __( 'This URL has already been added.', 'askany' ), array( 'status' => 400 ) );
		}

		// Insert new external source
		$result = $wpdb->insert(
			$this->external_sources_table,
			array(
				'url' => $url,
				'status' => 'pending',
				'follow_links' => $follow_links,
				'include_paths' => $include_paths,
				'exclude_paths' => $exclude_paths,
				'include_selectors' => $include_selectors,
				'exclude_selectors' => $exclude_selectors,
				'created_at' => current_time( 'mysql' ),
				'updated_at' => current_time( 'mysql' )
			),
			array( '%s', '%s', '%d', '%s', '%s', '%s', '%s', '%s', '%s' )
		);

		if ( $result === false ) {
			return new \WP_Error( 'insert_failed', __( 'Failed to add external source.', 'askany' ), array( 'status' => 500 ) );
		}

		$source_id = $wpdb->insert_id;

		// Schedule background processing
		wp_schedule_single_event( time() + 5, 'askany_process_external_url', array( $source_id, $url ) );

		return new \WP_REST_Response( array(
			'success' => true,
			'message' => __( 'External source added successfully. Processing will begin shortly.', 'askany' ),
			'source_id' => $source_id
		), 201 );
	}

	/**
	 * Get all external sources.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response The response object.
	 */
	public function get_external_sources( $request ) {
		global $wpdb;

		$sources = $wpdb->get_results(
			"SELECT * FROM {$this->external_sources_table} ORDER BY created_at DESC"
		);

		// Process crawled URLs for display
		foreach ( $sources as $source ) {
			$source->crawled_urls_array = ! empty( $source->crawled_urls ) ? json_decode( $source->crawled_urls, true ) : array();
			$source->crawled_count = is_array( $source->crawled_urls_array ) ? count( $source->crawled_urls_array ) : 0;
		}

		return new \WP_REST_Response( array(
			'success' => true,
			'sources' => $sources
		) );
	}

	/**
	 * Delete an external source.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response|\WP_Error The response object or error.
	 */
	public function delete_external_source( $request ) {
		global $wpdb;

		$source_id = (int) $request->get_param( 'id' );

		// Get source details first
		$source = $wpdb->get_row( $wpdb->prepare(
			"SELECT * FROM {$this->external_sources_table} WHERE id = %d",
			$source_id
		) );

		if ( ! $source ) {
			return new \WP_Error( 'source_not_found', __( 'External source not found.', 'askany' ), array( 'status' => 404 ) );
		}

		// Delete the crawled pages and their vectors.
		$crawled_urls = ! empty( $source->crawled_urls ) ? json_decode( $source->crawled_urls, true ) : array();
		if ( is_array( $crawled_urls ) ) {
			$this->remove_pages( $crawled_urls );
		}

		// Delete the external source
		$result = $wpdb->delete(
			$this->external_sources_table,
			array( 'id' => $source_id ),
			array( '%d' )
		);

		if ( $result === false ) {
			return new \WP_Error( 'delete_failed', __( 'Failed to delete external source.', 'askany' ), array( 'status' => 500 ) );
		}

		do_action( 'askany_external_source_deleted', $source_id );

		return new \WP_REST_Response( array(
			'success' => true,
			'message' => __( 'External source deleted successfully.', 'askany' )
		) );
	}

	/**
	 * Get crawled URLs for a specific source.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response|\WP_Error The response object or error.
	 */
	public function get_crawled_urls( $request ) {
		global $wpdb;

		$source_id = (int) $request->get_param( 'id' );

		$source = $wpdb->get_row( $wpdb->prepare(
			"SELECT crawled_urls FROM {$this->external_sources_table} WHERE id = %d",
			$source_id
		) );

		if ( ! $source ) {
			return new \WP_Error( 'source_not_found', __( 'External source not found.', 'askany' ), array( 'status' => 404 ) );
		}

		$crawled_urls = ! empty( $source->crawled_urls ) ? json_decode( $source->crawled_urls, true ) : array();

		return new \WP_REST_Response( array(
			'success' => true,
			'urls' => is_array( $crawled_urls ) ? $crawled_urls : array()
		) );
	}

	/**
	 * Reprocess an external source.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response|\WP_Error The response object or error.
	 */
	public function reprocess_external_source( $request ) {
		global $wpdb;

		$source_id = (int) $request->get_param( 'id' );

		$source = $wpdb->get_row( $wpdb->prepare(
			"SELECT * FROM {$this->external_sources_table} WHERE id = %d",
			$source_id
		) );

		if ( ! $source ) {
			return new \WP_Error( 'source_not_found', __( 'External source not found.', 'askany' ), array( 'status' => 404 ) );
		}

		// Update status to pending
		$wpdb->update(
			$this->external_sources_table,
			array( 'status' => 'pending', 'error_message' => '', 'updated_at' => current_time( 'mysql' ) ),
			array( 'id' => $source_id ),
			array( '%s', '%s', '%s' ),
			array( '%d' )
		);

		// Schedule background processing
		wp_schedule_single_event( time() + 5, 'askany_process_external_url', array( $source_id, $source->url ) );

		return new \WP_REST_Response( array(
			'success' => true,
			'message' => __( 'External source reprocessing started.', 'askany' )
		) );
	}

	/**
	 * Check processing status of external sources.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response The response object.
	 */
	public function check_processing_status( $request ) {
		global $wpdb;

		// Get sources that are pending or processing
		$processing_sources = $wpdb->get_results(
			"SELECT id, url, status FROM {$this->external_sources_table} 
			WHERE status IN ('pending', 'processing')"
		);
		
		// Get total sources count
		$total_sources = $wpdb->get_var(
			"SELECT COUNT(*) FROM {$this->external_sources_table}"
		);

		return new \WP_REST_Response( array(
			'success' => true,
			'processing_sources' => $processing_sources,
			'total_sources' => (int) $total_sources
		) );
	}

	/**
	 * Process external URL in background.
	 *
	 * @param int    $source_id The source ID.
	 * @param string $url       The URL to process.
	 * @return void
	 */
	public function process_external_url_background( $source_id, $url ) {
		global $wpdb;

		// Update status to processing with initial progress
		$wpdb->update(
			$this->external_sources_table,
			array( 
				'status' => 'processing',
				'processing_progress' => json_encode(array(
					'current' => 0,
					'total' => 1,
					'message' => 'Starting crawl...',
					'current_url' => $url
				)),
				'updated_at' => current_time( 'mysql' )
			),
			array( 'id' => $source_id ),
			array( '%s', '%s', '%s' ),
			array( '%d' )
		);

		try {
			// Get source configuration
			$source = $wpdb->get_row( $wpdb->prepare(
				"SELECT * FROM {$this->external_sources_table} WHERE id = %d",
				$source_id
			) );

			if ( ! $source ) {
				throw new \Exception( 'Source not found' );
			}

			// Process the URL with progress tracking
			$result = $this->crawl_and_index_url( $source, $source_id );

			if ( $result['success'] ) {
				// Update status to completed
				$wpdb->update(
					$this->external_sources_table,
					array(
						'status' => 'completed',
						'title' => $result['title'],
						'crawled_urls' => json_encode( $result['crawled_urls'] ),
						'last_crawled' => current_time( 'mysql' ),
						'error_message' => '',
						'processing_progress' => json_encode(array(
							'current' => count($result['crawled_urls']),
							'total' => count($result['crawled_urls']),
							'message' => 'Completed successfully',
							'current_url' => ''
						)),
						'updated_at' => current_time( 'mysql' )
					),
					array( 'id' => $source_id ),
					array( '%s', '%s', '%s', '%s', '%s', '%s', '%s' ),
					array( '%d' )
				);
				do_action( 'askany_external_source_saved', $source_id );
			} else {
				// Update status to failed
				$wpdb->update(
					$this->external_sources_table,
					array(
						'status' => 'failed',
						'error_message' => $result['error'],
						'processing_progress' => '',
						'updated_at' => current_time( 'mysql' )
					),
					array( 'id' => $source_id ),
					array( '%s', '%s', '%s', '%s' ),
					array( '%d' )
				);
			}
		} catch ( \Exception $e ) {
			// Update status to failed
			$wpdb->update(
				$this->external_sources_table,
				array(
					'status' => 'failed',
					'error_message' => $e->getMessage(),
					'processing_progress' => '',
					'updated_at' => current_time( 'mysql' )
				),
				array( 'id' => $source_id ),
				array( '%s', '%s', '%s', '%s' ),
				array( '%d' )
			);
		}
	}

	/**
	 * Crawl and index a URL.
	 *
	 * @param object $source The source configuration.
	 * @param int $source_id The source ID for progress tracking.
	 * @return array The result array with success status and data.
	 */
	private function crawl_and_index_url( $source, $source_id = null ) {
		global $wpdb;
		$crawled_urls = array();
		$title = '';

		try {
			// Update progress: Fetching main URL
			if ( $source_id ) {
				$this->update_processing_progress( $source_id, 0, 1, 'Fetching main URL...', $source->url );
			}
			
			// Fetch main URL content
			$main_content = $this->fetch_url_content( $source->url );
			
			if ( ! $main_content ) {
				return array(
					'success' => false,
					'error' => 'Failed to fetch main URL content'
				);
			}

			if ( function_exists( 'set_time_limit' ) ) {
				@set_time_limit( 600 ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged
			}
			$previous_urls = ! empty( $source->crawled_urls ) ? json_decode( $source->crawled_urls, true ) : array();
			$previous_urls = is_array( $previous_urls ) ? $previous_urls : array();
			$max_pages     = max( 1, (int) apply_filters( 'askany_external_max_pages', $this->max_pages, $source ) );

			$title = $main_content['title'];
			$crawled_urls[] = $source->url;

			// Update progress: Indexing main URL
			if ( $source_id ) {
				$this->update_processing_progress( $source_id, 1, 1, 'Indexing main URL...', $source->url );
			}
			
			// Index main URL content
			$this->store_page( $source->url, $main_content['title'], $main_content['content'] );

			// If follow_links is enabled, crawl linked pages
			if ( $source->follow_links ) {
				// Update progress: Discovering links
				if ( $source_id ) {
					$this->update_processing_progress( $source_id, 1, 1, 'Discovering linked pages...', $source->url );
				}
				
				$linked_urls = $this->discover_linked_urls( $source->url, $main_content['html'], $source );
				$total_urls = min( count( $linked_urls ), $max_pages - 1 );
				$current_index = 1; // Already processed main URL
				
				foreach ( array_slice( $linked_urls, 0, $max_pages - 1 ) as $linked_url ) {
					// Update progress: Processing linked URL
					if ( $source_id ) {
						$this->update_processing_progress( 
							$source_id, 
							$current_index, 
							$total_urls + 1, 
							sprintf( 'Processing page %d of %d...', $current_index + 1, $total_urls + 1 ),
							$linked_url 
						);
					}
					
					$linked_content = $this->fetch_url_content( $linked_url );
					
					if ( $linked_content ) {
						$crawled_urls[] = $linked_url;
						
						// Index linked URL content
						$this->store_page( $linked_url, $linked_content['title'], $linked_content['content'] );
					}

					$current_index++;
					
					// Small delay to be respectful
					usleep( 500000 ); // 0.5 seconds
				}
			}

			// Pages that were crawled before but are gone now.
			$this->remove_pages( array_diff( $previous_urls, $crawled_urls ) );

			return array(
				'success' => true,
				'title' => $title,
				'crawled_urls' => $crawled_urls
			);

		} catch ( \Exception $e ) {
			return array(
				'success' => false,
				'error' => $e->getMessage()
			);
		}
	}
	
	/**
	 * Update processing progress for a source.
	 *
	 * @param int $source_id The source ID.
	 * @param int $current Current progress count.
	 * @param int $total Total items to process.
	 * @param string $message Progress message.
	 * @param string $current_url Currently processing URL.
	 * @return void
	 */
	private function update_processing_progress( $source_id, $current, $total, $message, $current_url = '' ) {
		global $wpdb;
		
		$progress_data = array(
			'current' => $current,
			'total' => $total,
			'percentage' => $total > 0 ? round( ( $current / $total ) * 100 ) : 0,
			'message' => $message,
			'current_url' => $current_url,
			'timestamp' => current_time( 'mysql' )
		);
		
		$wpdb->update(
			$this->external_sources_table,
			array( 
				'processing_progress' => json_encode( $progress_data ),
				'updated_at' => current_time( 'mysql' )
			),
			array( 'id' => $source_id ),
			array( '%s', '%s' ),
			array( '%d' )
		);
	}

	/**
	 * Fetch content from a URL.
	 *
	 * @param string $url The URL to fetch.
	 * @return array|false The content array or false on failure.
	 */
	private function fetch_url_content( $url ) {
		// Only the site itself may be reached on a private/internal address; any other
		// host goes through wp_safe_remote_get (no internal IPs, also after redirects).
		$own_host = wp_parse_url( home_url(), PHP_URL_HOST );
		$fetch    = ( wp_parse_url( $url, PHP_URL_HOST ) === $own_host ) ? 'wp_remote_get' : 'wp_safe_remote_get';
		$response = $fetch( $url, array(
			'timeout' => $this->request_timeout,
			'user-agent' => 'AskAny Bot (+' . home_url() . ')',
			'headers' => array(
				'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
			)
		) );

		if ( is_wp_error( $response ) ) {
			return false;
		}

		$body = wp_remote_retrieve_body( $response );
		$status_code = wp_remote_retrieve_response_code( $response );

		if ( $status_code !== 200 || empty( $body ) ) {
			return false;
		}

		// Parse HTML content as UTF-8 (DOMDocument assumes ISO-8859-1 otherwise,
		// which turned "’" into "â€™" and emoji into garbage in the index).
		$dom = new \DOMDocument();
		@$dom->loadHTML( '<?xml encoding="UTF-8">' . $body );

		// Extract title
		$title_nodes = $dom->getElementsByTagName( 'title' );
		$title = $title_nodes->length > 0 ? $title_nodes->item( 0 )->textContent : '';

		// Extract main content (remove scripts, styles, etc.)
		$this->remove_unwanted_elements( $dom );
		$content = $this->extract_text_content( $dom );

		return array(
			'title' => trim( $title ),
			'content' => trim( $content ),
			'html' => $body
		);
	}

	/**
	 * Discover linked URLs from HTML content.
	 *
	 * @param string $base_url The base URL.
	 * @param string $html     The HTML content.
	 * @param object $source   The source configuration.
	 * @return array Array of discovered URLs.
	 */
	private function discover_linked_urls( $base_url, $html, $source ) {
		$dom = new \DOMDocument();
		@$dom->loadHTML( '<?xml encoding="UTF-8">' . $html );

		$links = $dom->getElementsByTagName( 'a' );
		$discovered_urls = array();
		$base_domain = parse_url( $base_url, PHP_URL_HOST );

		foreach ( $links as $link ) {
			$href = $link->getAttribute( 'href' );
			
			if ( empty( $href ) || $href === '#' ) {
				continue;
			}

			// Convert relative URLs to absolute
			$absolute_url = $this->make_absolute_url( $href, $base_url );
			
			if ( ! $absolute_url ) {
				continue;
			}

			if ( preg_match( '/^(mailto|tel|javascript|data):/i', $href ) ) {
				continue;
			}
			$absolute_url = $this->normalize_url( $absolute_url );

			// Only include URLs from the same domain
			$link_domain = parse_url( $absolute_url, PHP_URL_HOST );
			if ( $link_domain !== $base_domain ) {
				continue;
			}

			// Pages only: skip files and feeds, and the start page itself.
			$link_path = (string) parse_url( $absolute_url, PHP_URL_PATH );
			if ( preg_match( '/\.(css|js|json|xml|png|jpe?g|gif|webp|svg|ico|pdf|zip|mp4|mp3|woff2?|ttf)$/i', $link_path )
				|| preg_match( '#/(feed|wp-json|wp-admin|wp-content|wp-includes)(/|$)#i', $link_path )
				|| $absolute_url === $this->normalize_url( $base_url ) ) {
				continue;
			}

			// Apply include/exclude path filters
			if ( ! $this->should_include_url( $absolute_url, $source ) ) {
				continue;
			}

			$discovered_urls[] = $absolute_url;
		}

		return array_unique( $discovered_urls );
	}

	/**
	 * Make a URL absolute.
	 *
	 * @param string $url      The URL to make absolute.
	 * @param string $base_url The base URL.
	 * @return string|false The absolute URL or false on failure.
	 */
	private function make_absolute_url( $url, $base_url ) {
		// Already absolute
		if ( parse_url( $url, PHP_URL_SCHEME ) ) {
			return $url;
		}

		$base_parts = parse_url( $base_url );
		
		if ( ! $base_parts ) {
			return false;
		}

		// Protocol-relative URL
		if ( substr( $url, 0, 2 ) === '//' ) {
			return $base_parts['scheme'] . ':' . $url;
		}

		// Absolute path
		if ( substr( $url, 0, 1 ) === '/' ) {
			return $base_parts['scheme'] . '://' . $base_parts['host'] . 
				   ( isset( $base_parts['port'] ) ? ':' . $base_parts['port'] : '' ) . $url;
		}

		// Relative path
		$base_path = isset( $base_parts['path'] ) ? dirname( $base_parts['path'] ) : '';
		if ( $base_path === '.' ) {
			$base_path = '';
		}

		return $base_parts['scheme'] . '://' . $base_parts['host'] . 
			   ( isset( $base_parts['port'] ) ? ':' . $base_parts['port'] : '' ) . 
			   rtrim( $base_path, '/' ) . '/' . $url;
	}

	/**
	 * Check if a URL should be included based on include/exclude filters.
	 *
	 * @param string $url    The URL to check.
	 * @param object $source The source configuration.
	 * @return bool Whether the URL should be included.
	 */
	private function should_include_url( $url, $source ) {
		$path = parse_url( $url, PHP_URL_PATH );
		
		if ( ! $path ) {
			$path = '/';
		}

		// Check include paths
		if ( ! empty( $source->include_paths ) ) {
			$include_paths = array_filter( array_map( 'trim', explode( ',', $source->include_paths ) ) );
			$included = false;
			
			foreach ( $include_paths as $include_path ) {
				if ( strpos( $path, $include_path ) !== false ) {
					$included = true;
					break;
				}
			}
			
			if ( ! $included ) {
				return false;
			}
		}

		// Check exclude paths
		if ( ! empty( $source->exclude_paths ) ) {
			$exclude_paths = array_filter( array_map( 'trim', explode( ',', $source->exclude_paths ) ) );
			
			foreach ( $exclude_paths as $exclude_path ) {
				if ( strpos( $path, $exclude_path ) !== false ) {
					return false;
				}
			}
		}

		return true;
	}

	/**
	 * Remove unwanted elements from DOM.
	 *
	 * @param \DOMDocument $dom The DOM document.
	 * @return void
	 */
	private function remove_unwanted_elements( $dom ) {
		$unwanted_tags = array( 'script', 'style', 'nav', 'header', 'footer', 'aside', 'noscript' );
		
		foreach ( $unwanted_tags as $tag ) {
			$elements = $dom->getElementsByTagName( $tag );
			$to_remove = array();
			
			foreach ( $elements as $element ) {
				$to_remove[] = $element;
			}
			
			foreach ( $to_remove as $element ) {
				if ( $element->parentNode ) {
					$element->parentNode->removeChild( $element );
				}
			}
		}
	}

	/**
	 * Extract text content from DOM.
	 *
	 * @param \DOMDocument $dom The DOM document.
	 * @return string The extracted text content.
	 */
	private function extract_text_content( $dom ) {
		$body = $dom->getElementsByTagName( 'body' )->item( 0 );
		
		if ( ! $body ) {
			return '';
		}

		// Keep block structure: a line break after headings, paragraphs, list items, rows.
		$xpath = new \DOMXPath( $dom );
		foreach ( $xpath->query( '//body//*[self::p or self::div or self::section or self::article or self::li or self::tr or self::br or self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6 or self::td or self::th]' ) as $node ) {
			$node->appendChild( $dom->createTextNode( ( 'td' === $node->nodeName || 'th' === $node->nodeName ) ? ' | ' : "\n" ) );
		}

		$text = $body->textContent;

		// Clean up whitespace (keep single line breaks).
		$text = preg_replace( '/[ \t\x{00A0}]+/u', ' ', $text );
		$text = preg_replace( '/ *\n\s*/u', "\n", (string) $text );
		$text = trim( (string) $text );

		/**
		 * Maximum characters stored per crawled page.
		 *
		 * @param int $max Default 60000.
		 */
		$max = (int) apply_filters( 'askany_external_page_max_chars', 60000 );
		if ( $max > 0 && function_exists( 'mb_substr' ) && mb_strlen( $text ) > $max ) {
			$text = mb_substr( $text, 0, $max );
		}

		return $text;
	}

	/**
	 * Stable numeric index id for a crawled URL (the index column is BIGINT; a raw
	 * md5 string was cast to a tiny number, so different pages overwrote each other).
	 *
	 * @param string $url URL.
	 * @return int
	 */
	public static function url_object_id( $url ) {
		return (int) hexdec( substr( md5( (string) $url ), 0, 15 ) );
	}

	/**
	 * Normalise a crawled URL: no fragment, no trailing slash on paths.
	 *
	 * @param string $url URL.
	 * @return string
	 */
	private function normalize_url( $url ) {
		$url   = preg_replace( '/#.*$/', '', trim( (string) $url ) );
		$parts = wp_parse_url( $url );
		if ( empty( $parts['host'] ) ) {
			return $url;
		}
		$path = isset( $parts['path'] ) ? rtrim( $parts['path'], '/' ) : '';
		return ( $parts['scheme'] ?? 'https' ) . '://' . $parts['host']
			. ( isset( $parts['port'] ) ? ':' . $parts['port'] : '' )
			. ( '' === $path ? '/' : $path )
			. ( isset( $parts['query'] ) ? '?' . $parts['query'] : '' );
	}

	/**
	 * Store (or refresh) one crawled page in the content index. Changed text
	 * drops the old vectors so the page is embedded again.
	 *
	 * @param string $url     Page URL.
	 * @param string $title   Page title.
	 * @param string $content Page text.
	 * @return void
	 */
	private function store_page( $url, $title, $content ) {
		global $wpdb;
		$table     = $wpdb->prefix . 'askany_content_index';
		$emb_table = $wpdb->prefix . 'askany_embeddings';
		$object_id = self::url_object_id( $url );
		$content   = html_entity_decode( (string) $content, ENT_QUOTES | ENT_HTML5, 'UTF-8' );
		$title     = html_entity_decode( (string) $title, ENT_QUOTES | ENT_HTML5, 'UTF-8' );
		if ( '' === trim( $content ) ) {
			return;
		}
		$text = "Page: {$title}\nURL: {$url}\n\n{$content}";

		// Rows of this URL stored under an old (colliding) id.
		$stale = $wpdb->get_col( $wpdb->prepare( "SELECT id FROM {$table} WHERE object_type = 'external_url' AND url = %s AND object_id <> %d", $url, $object_id ) );
		foreach ( $stale as $stale_id ) {
			$wpdb->delete( $emb_table, array( 'content_id' => (int) $stale_id ), array( '%d' ) );
			$wpdb->delete( $table, array( 'id' => (int) $stale_id ), array( '%d' ) );
		}

		$existing = $wpdb->get_row( $wpdb->prepare( "SELECT id, content, title FROM {$table} WHERE object_type = 'external_url' AND object_id = %d", $object_id ), ARRAY_A );
		$now      = current_time( 'mysql' );
		if ( $existing ) {
			if ( $existing['content'] === $text && $existing['title'] === $title ) {
				$wpdb->update( $table, array( 'updated_at' => $now ), array( 'id' => (int) $existing['id'] ), array( '%s' ), array( '%d' ) );
				return;
			}
			$wpdb->delete( $emb_table, array( 'content_id' => (int) $existing['id'] ), array( '%d' ) );
			$wpdb->update(
				$table,
				array(
					'content'            => $text,
					'title'              => $title,
					'url'                => $url,
					'updated_at'         => $now,
					'has_embedding'      => 0,
					'embedding_status'   => null,
					'embedding_error'    => null,
					'embedding_attempts' => 0,
				),
				array( 'id' => (int) $existing['id'] )
			);
			return;
		}
		$wpdb->insert(
			$table,
			array(
				'object_id'   => $object_id,
				'object_type' => 'external_url',
				'content'     => $text,
				'title'       => $title,
				'url'         => $url,
				'created_at'  => $now,
				'updated_at'  => $now,
			),
			array( '%d', '%s', '%s', '%s', '%s', '%s', '%s' )
		);
	}

	/**
	 * Remove crawled pages (and their vectors) from the index.
	 *
	 * @param array $urls URLs.
	 * @return void
	 */
	private function remove_pages( array $urls ) {
		global $wpdb;
		$table     = $wpdb->prefix . 'askany_content_index';
		$emb_table = $wpdb->prefix . 'askany_embeddings';
		foreach ( array_unique( array_filter( $urls ) ) as $url ) {
			foreach ( array_unique( array( $url, $this->normalize_url( $url ) ) ) as $u ) {
				$ids = $wpdb->get_col( $wpdb->prepare( "SELECT id FROM {$table} WHERE object_type = 'external_url' AND url = %s", $u ) );
				foreach ( $ids as $id ) {
					$wpdb->delete( $emb_table, array( 'content_id' => (int) $id ), array( '%d' ) );
					$wpdb->delete( $table, array( 'id' => (int) $id ), array( '%d' ) );
				}
			}
		}
	}

	/**
	 * Get external sources statistics.
	 *
	 * @return array Statistics array.
	 */
	public function get_stats() {
		global $wpdb;

		return array(
			'total_sources' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table}" ),
			'completed_sources' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table} WHERE status = 'completed'" ),
			'processing_sources' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table} WHERE status = 'processing'" ),
			'failed_sources' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table} WHERE status = 'failed'" ),
			'total_crawled_urls' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->prefix}askany_content_index WHERE object_type = 'external_url'" ),
			'last_crawled' => $wpdb->get_var( "SELECT MAX(last_crawled) FROM {$this->external_sources_table}" )
		);
	}
}