<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org (initial MediaJob class
 *      and subclasses based on work of Pooja Mishra for her master's)
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library\media_jobs;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\models as M;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\storage_formats\IndexShard;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\models\GroupModel;
use seekquarry\yioop\controllers\CrawlController;

/**
 * A media job to compute trending terms from the feed search sources, and
 * to generate a list of top feed items for the landing page for the
 * different subsearches displlayed on the landing page.
 */
class TrendingHighlightsJob extends MediaJob
{
    /**
     * Mamimum number of trending values to download in one try
     */
    const MAX_VALUES_ONE_GO = 100;
    /**
     * Time in current epoch when feeds last updated
     * @var int
     */
    public $update_time;
    /**
     * Datasource object used to run db queries related to fes items
     * (for storing and updating them)
     * @var object
     */
    public $db;
    /**
     * Initializes the last update time to far in the past so, feeds will get
     * immediately updated. Sets up connect to DB to store feeds items, and
     * makes it so the same media job runs both on name server and client
     * Media Updaters
     */
    public function init()
    {
        $this->update_time = 0;
        $db_class = C\NS_DATASOURCES . ucfirst(C\p('DBMS')). "Manager";
        $this->db = new $db_class();
        $this->db->connect();
        C\nsconddefine("TRENDING_UPDATE_INTERVAL", C\ONE_HOUR);
    }
    /**
     * Only update if its been more than an hour since the last update
     *
     * @return bool whether its been an hour since the last update
     */
    public function checkPrerequisites()
    {
        $time = time();
        $something_updated = false;
        $delta = $time - $this->update_time;
        if ($delta > C\TRENDING_UPDATE_INTERVAL) {
            $this->update_time = $time;
            L\crawlLog("Performing trending value update");
            return true;
        }
        L\crawlLog("Time since last update not exceeded, skipping trending value
            update");
        return false;
    }
    /**
     * For now trending values only computed on Namesevers (as specific
     * to the database there) so is done in prepareTasks
     */
    public function nondistributedTasks()
    {
        $this->prepareTasks();
    }
    /**
     * (Run on name server)
     */
    public function prepareTasks()
    {
        $db = $this->db;
        $sql = "SELECT * FROM MEDIA_SOURCE WHERE (TYPE='trending_value')";
        $result = $db->execute($sql);
        $trending_value_sites = [];
        while ($trending_value_site = $db->fetchArray($result)) {
            $aux_parts = explode("###",
                html_entity_decode($trending_value_site['AUX_INFO'],
                ENT_QUOTES));
            if (!isset($aux_parts[6])) {
                continue;
            }
            $trending_value_site['CATEGORY_GROUP'] = $aux_parts[5];
            $trending_value_site['TREND_VALUE_REGEX'] = $aux_parts[6];
            $trending_value_sites[] = $trending_value_site;
        }
        $this->tasks = $trending_value_sites;
        L\crawlLog("----This media updater is responsible for the trending ".
            "value_sites");
        $i = 1;
        foreach ($trending_value_sites as $trending_value_site) {
            L\crawlLog("----  $i. " . $trending_value_site["NAME"]);
            $i++;
        }
        $num_trending_sites = count($trending_value_sites);
        $trending_values_one_go = self::MAX_VALUES_ONE_GO;
        $limit = 0;
        while ($limit < $num_trending_sites) {
            $trending_values_batch = array_slice($trending_value_sites, $limit,
                $trending_values_one_go);
            $this->updateTrendingValuesOneGo($trending_values_batch);
            $limit += $trending_values_one_go;
        }
        L\crawlLog("----Computing Landing Page Highlights");
        $this->computeLandingHighlights();
    }
    /**
     * For each of a supplied list of podcast associative arrays,
     * downloads the non-expired media for that podcast to the wiki folder
     * specified.
     *
     * @param array $trending_sites an array of associative arrays of info
     *      about values to download
     * @param int $age oldest age items to consider for download
     * @param boolean $test_mode if true then rather then updating items in
     *  wiki, returns as a string summarizing the results of the downloads
     *  that would occur as part of updating the podcast
     * @return mixed either true, or if $test_mode is true then the results
     *      as a string of the operations involved in downloading the podcasts
     */
    public function updateTrendingValuesOneGo($trending_sites,
        $age = C\ONE_WEEK, $test_mode = false)
    {
        $db =$this->db;
        $test_results = "";
        $log_function = function ($msg, $log_tag = "pre class='source-test'")
            use (&$test_results, $test_mode) {
            $close_tag= preg_split("/\s+/",$log_tag)[0];
            if ($test_mode) {
                $test_results .= "<$log_tag>$msg</$close_tag>\n";
            } else {
                L\crawlLog($msg);
            }
        };
        $trending_sites = FetchUrl::getPages($trending_sites, false, 0, null,
            "SOURCE_URL", CrawlConstants::PAGE, false, null, true);
        $insert_sql = "INSERT INTO TRENDING_TERM (TERM, OCCURRENCES, " .
            "UPDATE_PERIOD, TIMESTAMP, LANGUAGE, CATEGORY) VALUES " .
            "(?, ?, ?, ?, ?, ?)";
        $time = time();
        foreach ($trending_sites as $trending_site) {
            $language = $trending_site['LANGUAGE'];
            $category = $trending_site['CATEGORY'];
            if (empty($trending_site[CrawlConstants::PAGE])) {
                $log_function(
                    "...No data in {$trending_site['NAME']} skipping.", "h3");
                continue;
            }
            $log_function("----Updating {$trending_site['NAME']}.", "h3");
            // strip namespaces
            $regex = $trending_site['TREND_VALUE_REGEX'];
            $page = $trending_site[CrawlConstants::PAGE];
            if (preg_match_all($regex, $page, $matches)) {
                $trending_matches = [];
                if (isset($matches[2])) {
                    $trending_matches = array_combine($matches[1], $matches[2]);
                } else if(isset($matches[1])) {
                    $trending_matches =[[$trending_site['CATEGORY_GROUP']
                        => $matches[1]]];
                }
                foreach ($trending_matches as $name => $value) {
                    $db->execute($insert_sql, [$name, $value,
                        C\TRENDING_UPDATE_INTERVAL,
                        $time, $language, $category]);
                }
            } else {
                $log_function("----could not parse an values from page.");
            }
        }
        return ($test_mode) ? $test_results : true;
    }
    /**
     * Computes arrays of data used to display the news and trending highlights
     * on the search landing page. Saves these precomputed arrays to
     * C\TRENDS_DIR under a file for each locale
     */
    public function computeLandingHighlights()
    {
        $locale_model = new M\LocaleModel();
        $source_model = new M\SourceModel();
        $subsearches =  $source_model->getSubsearches();
        $phrase_model = new M\PhraseModel();
        $crawl_model = new M\CrawlModel();
        $machine_model = new M\MachineModel();
        $verticals_model = new M\SearchverticalsModel;
        $trending_model = new M\TrendingModel;
        $locale_list = $locale_model->getLocaleList();
        $locale = C\p('DEFAULT_LOCALE');
        $locale_major = (explode("-", $locale))[0];
        foreach ($locale_list as $language) {
            $locale_tag = $language['LOCALE_TAG'];
            $locale_tag_major = (explode("-", $locale_tag))[0];
            $highlights_file = C\TRENDS_DIR . "/highlights_$locale_tag.txt";
            $num_random_trends = 9;
            $highlights = [];
            foreach ($subsearches as $subsearch) {
                if (preg_match("/highlight:(\d+)\b/ui",
                    $subsearch["DEFAULT_QUERY"] ?? "", $highlight_match)) {
                    $subsearch['PRIORITY'] = (empty($highlight_match[1])) ? 0 :
                        floatval($highlight_match[1]);
                    if (preg_match("/trending:(\w+):(\w+)\b/ui",
                        $subsearch["DEFAULT_QUERY"], $trending_parts)) {
                        $subsearch['CATEGORY'] = $trending_parts[1] ?? "";
                        if (empty($trending_parts[2])) {
                            $subsearch['ORDER'] = "SCORE_DESC";
                        } else {
                            $subsearch['ORDER'] = $trending_parts[2];
                        }
                    }
                    $highlights[] = $subsearch;
                }
            }
            usort($highlights, function($a, $b) {
                if ($a['PRIORITY'] == $b['PRIORITY']) {
                    return 0;
                }
                return ($a['PRIORITY'] > $b['PRIORITY']) ? -1 : 1;
            });
            $landing_highlight_data = [];
            $queue_servers = $machine_model->getQueueServerUrls();
            foreach ($highlights as $highlight) {
                if (!empty($highlight['CATEGORY'])) {
                    $landing_highlight_data[] = [
                        "NAME" => $phrase_model->translateDb(
                            $highlight['LOCALE_STRING'], $locale_tag),
                        'TYPE' => 'random_terms',
                        'CATEGORY' => $highlight['CATEGORY'],
                        "ORDER" => $highlight['ORDER'],
                        'DATA' => $trending_model->randomTrends(
                            $locale_tag, $highlight['CATEGORY'],
                            $num_random_trends)
                    ];
                } else {
                    $query = preg_replace("/trending:\w+|highlight:\w+/ui",
                        '', $highlight['DEFAULT_QUERY']);
                    if (preg_match("/\bm:(\d+)\b/",
                        $highlight['INDEX_IDENTIFIER'], $matches)) {
                        $mix = $crawl_model->getCrawlMix($matches[1]);
                        $query = $phrase_model->rewriteMixQuery($query, $mix);
                    } else {
                        $query = ($highlight['INDEX_IDENTIFIER'] == -1) ?
                            "site:all" : $highlight['INDEX_IDENTIFIER'] . " " .
                            $query;
                    }
                    $query = preg_replace('/lang:default-major/ui', "lang:" .
                        $locale_tag_major, $query);
                    $query = preg_replace('/lang:default/u',
                        "lang:" . $locale_tag,
                        $query);
                    $query_results = $phrase_model->getPhrasePageResults(
                        $query, 0, 5, false, $verticals_model, true, 0,
                        $queue_servers);
                    $pages = $query_results['PAGES'] ?? [];
                    $landing_highlight_data[] = [
                        "FOLDER_NAME" => $highlight['FOLDER_NAME'],
                        "NAME" => $phrase_model->translateDb(
                            $highlight['LOCALE_STRING'], $locale_tag),
                        'TYPE' => 'feed',
                        'DATA' => $pages
                    ];
                }
            }
            set_error_handler(null);
            if (!file_exists(C\TRENDS_DIR)) {
                @mkdir(C\TRENDS_DIR);
                @chmod(C\TRENDS_DIR, 0777);
            }
            file_put_contents($highlights_file,
                serialize($landing_highlight_data));
            @chmod($highlights_file, 0777);
            restore_error_handler();
        }
    }
}
