<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\executables;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\data_structures\BloomFilterFile;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\storage_formats\IndexArchiveBundle;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\IndexManager;
use seekquarry\yioop\library\storage_formats\IndexShard;
use seekquarry\yioop\library\data_structures\PackedTableTools;
use seekquarry\yioop\library\storage_formats\PartitionDocumentBundle;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\library\storage_formats\WebArchiveBundle;
use seekquarry\yioop\library\media_jobs\FeedsUpdateJob;
use seekquarry\yioop\controllers\AdminController;
use seekquarry\yioop\models as M;
use seekquarry\yioop\models\ImpressionModel;

if (php_sapi_name() != 'cli' ||
    defined("seekquarry\\yioop\\configs\\IS_OWN_WEB_SERVER")) {
    echo "BAD REQUEST"; exit();
}
/** This tool does not need logging*/
$_SERVER["LOG_TO_FILES"] = false;
/** USE_CACHE false rules out file cache as well*/
$_SERVER["USE_CACHE"] = false;
/**  For crawlHash, crawlHashWord function */
require_once __DIR__."/../library/Utility.php";
require_once C\BASE_DIR . '/configs/Config.php';
if (!C\PROFILE) {
    echo "Please configure the search engine instance by visiting" .
        "its web interface on localhost.\n";
    exit();
}
ini_set("memory_limit", C\ARC_TOOL_MEMORY_LIMIT);   /*reading in a whole
    shard might take a fair bit of memory
*/
/*
 * We'll set up multi-byte string handling to use UTF-8
 */
mb_internal_encoding("UTF-8");
mb_regex_encoding("UTF-8");
/**
 * Command line program that allows one to examine the content of
 * the WebArchiveBundles and IndexArchiveBundles of Yioop crawls.
 * To see all of the available command run it from the command line
 * with a syntax like:
 *
 * php ArcTool.php
 *
 * @author Chris Pollett (non-yioop archive code derived from earlier
 *     stuff by Shawn Tice)
 */
class ArcTool extends DictionaryUpdater implements CrawlConstants
{
    /**
     * The maximum number of documents the ArcTool list function
     * will read into memory in one go.
     */
    const MAX_BUFFER_DOCS = 200;
    /**
     * The maximum number of documents the ArcTool will rebuild/migrate in
     * one go
     */
    const MAX_REBUILD_DOCS = 8000;
    /**
     * Initializes the ArcTool, for now does nothing
     */
    public function __construct()
    {
    }
    /**
     * Runs the ArcTool on the supplied command line arguments
     */
    public function start()
    {
        global $argv;
        if (!isset($argv[1]) || (!isset($argv[2]) &&
            !in_array($argv[1], ['list', 'user-impressions'], true)) ||
            (in_array($argv[1], ['dict', 'inject', 'make-filter'], true)
            && !isset($argv[3]) ) || ($argv[1] === 'doc-lookup'
            && !isset($argv[4]) ) ) {
            $this->usageMessageAndExit();
        }
        if (!in_array($argv[1], [ "check-filter", "make-filter", "list"])) {
            if (!in_array($argv[1], ["inject"])) {
                $path = UrlParser::getDocumentFilename($argv[2]);
                if ($path == $argv[2] && !file_exists($path)) {
                    $path = C\CACHE_DIR . "/" . $path;
                    if (!file_exists($path)) {
                        $path = C\ARCHIVES_DIR . "/" . $argv[2];
                    }
                }
                $kind = $this->getArchiveKind($path);
                if ($kind == "DoubleIndexBundle" && !in_array(
                    $argv[1], ["info", "migrate"]) ) {
                    $bundle_num = $argv[3];
                    unset($argv[3]);
                    $argv = array_values($argv);
                    $path .= "-$bundle_num";
                }
            } else if (is_numeric($argv[2])) {
                    $path = $argv[2];
            } else {
                $this->usageMessageAndExit();
            }
        }
        switch ($argv[1]) {
            case "check-filter":
                $this->checkFilter($argv[2], $argv[3]);
                break;
            case "count":
                $argv[3] ??= false;
                $this->outputCountBundle($path, $argv[3]);
                break;
            case "doc-lookup":
                $this->outputDocLookup($argv[2], intval($argv[3]),
                    intval($argv[4]));
                break;
            case "dict":
                if (isset($argv[4]) && $argv[4] == 'details') {
                    $argv[4] = -1;
                    $argv[5] = -1;
                    $argv[6] = "details";
                }
                $argv[4] = (isset($argv[4])) ? intval($argv[4]) : -1;
                $argv[5] = (isset($argv[5])) ? intval($argv[5]) : -1;
                $argv[6] = (!empty($argv[6]) && $argv[6] == "details") ?
                    true : false;
                $this->outputDictInfo($path, $argv[3], $argv[4], $argv[5],
                    $argv[6]);
                break;
            case "info":
                $this->outputInfo($path);
                break;
            case "inject":
                $this->inject($path, $argv[3]);
                break;
            case "list":
                $this->outputArchiveList();
                break;
            case "make-filter":
                $argv[4] ??= -1;
                $this->makeFilter($argv[2], $argv[3], $argv[4]);
                break;
            case "migrate":
                $argv[3] ??= 1;
                $this->migrateIndexArchive($path, $argv[3]);
                break;
            case "partition":
                $argv[3] ??= 0;
                $this->outputPartitionInfo($path, $argv[3]);
                break;
            case "fix-partition":
                $argv[3] ??= 0;
                $argv[4] ??= -1;
                $this->fixPartitionIndexes($path, $argv[3], $argv[4]);
                break;
            case "rebuild":
                $argv[3] ??= 0;
                $argv[4] ??= 1;
                $argv[5] ??= -1;
                $this->rebuildIndexBundle($path, $argv[3], true, $argv[4],
                    $argv[5]);
                break;
            case "remerge":
                $argv[3] ??= 0;
                $argv[4] ??= 1;
                $argv[5] ??= -1;
                $this->rebuildIndexBundle($path, $argv[3], false, $argv[4],
                    $argv[5]);
                break;
            case "show":
                if (!isset($argv[3])) {
                    $this->usageMessageAndExit();
                }
                $argv[4] ??= 1;
                $this->outputShowPages($path, $argv[3], $argv[4]);
                break;
            case "user-impressions":
                $key       = isset($argv[2]) ? (string)$argv[2] : null;
                $partition = isset($argv[3]) ? (int)$argv[3] : null;
                $limit     = isset($argv[4]) ? (int)$argv[4] : null;
                $this->displayLogRecords($key, $partition, $limit);
                break;
            case "item-impressions":
                $itemId    = isset($argv[2]) ? (int)$argv[2] : null;
                $partition = isset($argv[3]) ? (int)$argv[3] : null;
                $limit     = isset($argv[4]) ? (int)$argv[4] : null;
                if ($itemId === null) {
                    $this->usageMessageAndExit();
                }
                $this->displayLogRecordsByItemId($itemId, $partition, $limit);
                break;
            case "user-items":
                if (!isset($argv[2]) || !is_numeric($argv[2])) {
                    $this->usageMessageAndExit();
                }
                $this->getItemIdByUser((int)$argv[2]);
                break;
            case "group-items":
                if (!isset($argv[2]) || !is_numeric($argv[2])) {
                    $this->usageMessageAndExit();
                }
                $this->getItemIdByGroup((int)$argv[2]);
                break;
            case "type-items":
                if (!isset($argv[2])) {
                    $this->usageMessageAndExit();
                }
                $this->getItemIdByType($argv[2]);
                break;
            default:
                $this->usageMessageAndExit();
        }
    }
    /**
     * Lists the Web or IndexArchives in the crawl directory
     */
     public function outputArchiveList()
     {
        $yioop_pattern = C\CACHE_DIR . "/{" .
            self::double_index_base_name . "," . self::index_data_base_name .
            "}*";
        $archives = glob($yioop_pattern, GLOB_BRACE);
        $archives_found = false;
        if (is_array($archives) && count($archives) > 0) {
            $archives_found = true;
            echo "\nFound Yioop Archives:\n";
            echo "=====================\n";
            foreach ($archives as $archive_path) {
                $name = $this->getArchiveName($archive_path);
                echo $name . " ";
                $archive_type = $this->getArchiveKind($archive_path);
                if (in_array($archive_type, ["FeedArchiveBundle",
                    "DoubleIndexBundle", "IndexArchiveBundle",])) {
                    $bundle_class = C\NS_LIB . $archive_type;
                    $info = $bundle_class::getArchiveInfo($archive_path);
                    if (in_array(substr($info["DESCRIPTION"], 0, 2),
                        ['s:', 'a:'])) {
                        $info = unserialize($info["DESCRIPTION"]);
                    }
                    echo $info["DESCRIPTION"];
                }
                echo "\n";
            }
        }
        $nonyioop_pattern = C\ARCHIVES_DIR . "/*/arc_description.ini";
        $archives = glob($nonyioop_pattern);
        if (is_array($archives) && count($archives) > 0 ) {
            $archives_found = true;
            echo "\nFound Non-Yioop Archives:\n";
            echo "=========================\n";
            foreach ($archives as $archive_path) {
                $len = strlen("/arc_description.ini");
                $path = substr($archive_path, 0, -$len);
                echo $this->getArchiveName($path)."\n";
            }
        }
        if (!$archives_found) {
            echo "No archives currently in crawl directory \n";
        }
        echo "\n";
     }
    /**
     * Determines whether the supplied path is a WebArchiveBundle,
     * an IndexArchiveBundle, DoubleIndexBundle, or non-Yioop Archive.
     * Then outputs to stdout header information about the
     * bundle by calling the appropriate sub-function.
     *
     * @param string $archive_path The path of a directory that holds
     *     WebArchiveBundle,IndexArchiveBundle, or non-Yioop archive data
     */
    public function outputInfo($archive_path)
    {
        $bundle_name = $this->getArchiveName($archive_path);
        echo "Bundle Name: " . $bundle_name."\n";
        $archive_type = $this->getArchiveKind($archive_path);
        echo "Bundle Type: " . $archive_type."\n";
        if ($archive_type === false) {
            $this->badFormatMessageAndExit($archive_path);
        }
        if (in_array($archive_type, ["DoubleIndexBundle", "FeedDocumentBundle",
            "IndexDocumentBundle",])) {
            $call = "outputInfo" . $archive_type;
            $archive_name = C\NS_LIB . $archive_type;
            $info = $archive_name::getArchiveInfo($archive_path);
            $this->$call($info, $archive_path);
        }
    }
    /**
     * Outputs the $doc_map_index'th document from the $partition partition
     * of the IndexDocumentBundle in folder $index_name
     *
     * @param string $index_name folder containing an IndexDocumentBundle
     *  DoubleIndexBundle or FeedDocumentBundle
     * @param int $partition which partition to do the lookup in
     * @param int $doc_map_index index of which document to lookup
     * @return array associative array of field => values associated with
     *  document
     */
    public function outputDocLookup($index_name, $partition, $doc_map_index)
    {
        if (substr($index_name, 0, strlen(self::index_data_base_name)) ==
            self::index_data_base_name) {
            $index_name = substr($index_name,
                strlen(self::index_data_base_name));
        }
        if (substr($index_name, 0, strlen(self::double_index_base_name)) ==
            self::double_index_base_name) {
            $index_name = substr($index_name,
                strlen(self::double_index_base_name));
        }
        $index = IndexManager::getIndex($index_name);
        $base_folder = $index->getPartitionBaseFolder($partition);
        $doc_map_filename = $base_folder . "/" .
            IndexDocumentBundle::DOC_MAP_FILENAME;
        $doc_map_tools = $index->doc_map_tools;
        $entry = $doc_map_tools->findEntryAtIndexTableName($doc_map_filename,
            $doc_map_index);
        $docid_len = IndexDocumentBundle::DOCID_LEN;
        $termsfilter_len = IndexDocumentBundle::TERMSFILTER_LEN;
        $doc_key = substr($entry, 0, $docid_len);
        $entry = (strlen($entry) >= ($docid_len + $termsfilter_len + 1) &&
            $entry[$docid_len] == 't') ?
            substr($entry, $docid_len + $termsfilter_len + 1) :
            substr($entry, $docid_len);
        $doc_map_tools = $index->doc_map_tools;
        echo "Doc Key: " . L\toHexString($doc_key) . "\n";
        echo "Partition: $partition\n";
        echo "Doc Map Entries\n------------\n";
        $doc_info = $doc_map_tools->unpack($entry);
        $first_row = array_shift($doc_info);
        $second_row = array_shift($doc_info);
        if (!empty($first_row)) {
            list($num_words, $score) = array_values($first_row);
            echo "Number of Words in Document: $num_words\n";
            echo "Document Score: $score\n";
        }
        $num_scores = count($doc_info);
        if (!empty($second_row)) {
            list($title_length, $num_description_scores) = array_values(
                $second_row);
            echo "Title Length $title_length\n";
        }
        $score_index = 0;
        if ($num_description_scores > 0) {
            echo "Description Scores\n-------------------\n";
            while ($score_index < $num_description_scores) {
                $doc_row = $doc_info[$score_index];
                $score_index++;
                echo "Pos: " . $doc_row['POS'] .
                    " Score: ". $doc_row['SCORE']. "\n";
            }
        }
        if ($score_index < $num_scores) {
            echo "User Rank Scores\n-------------------\n";
            while ($score_index < $num_scores) {
                $doc_row = $doc_info[$score_index];
                echo "Score Index: " . $score_index .
                    " Score: ". $doc_row['SCORE'] . "\n";
                $score_index++;
            }
        }
        echo "Doc:\n----\n";
        $document = $index->documents->get($doc_key, $partition);
        var_dump($document);
    }
    /**
     * Prints the dictionary records for a word in an IndexDocumentBundle
     *
     * @param string $archive_path the path of a directory that holds
     *     an IndexArchiveBundle
     * @param string $word to look up dictionary record for
     * @param int $start_record first record to list out
     * @param int $num_records max records to list our
     * @param bool $details whether to show posting list details or not
     */
    public function outputDictInfo($archive_path, $word, $start_record,
        $num_records, $details)
    {
        $bundle_num = -1;
        if (preg_match("/\-\d$/", $archive_path)) {
            $bundle_num = substr($archive_path, -1);
            $archive_path = substr($archive_path, 0, -2);
        }
        $bundle_name = $this->getArchiveName($archive_path);
        echo "\nBundle Name: $bundle_name\n";
        $archive_type = $this->getArchiveKind($archive_path);
        echo "Bundle Type: $archive_type\n";
        if (!in_array($archive_type, ["FeedDocumentBundle",
            "DoubleIndexBundle", "IndexDocumentBundle",])) {
            $this->badFormatMessageAndExit($archive_path, "index");
        }
        preg_match("/\d+$/", $archive_path, $matches);
        $index_timestamp = (isset($matches[0])) ? $matches[0] : 0;
        if (isset($bundle_num) && $bundle_num >= 0) {
            $index_timestamp .= "-$bundle_num";
        } else if ($bundle_name == "IndexDataFeed") {
            $index_timestamp = "feed";
        }
        $hash_key = L\canonicalTerm($word);
        $start_time = microtime(true);
        echo "Looking up in dictionary:\n";
        echo " Key: " . $hash_key . "\n";
        echo " Key (in Hex): ". L\toHexString($hash_key) . "\n";
        $info = IndexManager::getWordInfo($index_timestamp, $hash_key,
            -1);
        $index = IndexManager::getIndex($index_timestamp);
        echo "Dictionary Lookup Time:" . L\changeInMicrotime($start_time)
            . "\n";
        if (!$info) {
            echo " Key not found\n";
            exit();
        }
        $found = true;
        echo "\nBundle Dictionary Entries for '$word':\n";
        echo "====================================\n";
        $i = 0;
        if ($index->archive_info['VERSION'] < "4.0") {
            echo "Cannot compute results for non-migrated indexes.\n";
            echo "Please use ArcTool's migrate command to upgrade index\n";
        }
        foreach ($info['ROWS'] as $record) {
            if ($start_record < 0 || $record['PARTITION'] >= $start_record) {
                echo "RECORD: $i\n";
                echo "PARTITION: {$record['PARTITION']}\n";
                echo "NUMBER OF DOCS: {$record['NUM_DOCS']}\n";
                $postings_offset = (empty($record['POSTINGS_OFFSET'])) ?
                    -1: $record['POSTINGS_OFFSET'];
                echo "POSTINGS_OFFSET: $postings_offset\n";
                $postings_len = (empty($record['POSTINGS_LEN']))?
                    -1 : $record['POSTINGS_LEN'];
                echo "POSTINGS_LEN: $postings_len\n\n";
                $is_postings_array = isset($record['POSTINGS']) &&
                    is_array($record['POSTINGS']);
                if ($postings_offset == -1 && !$is_postings_array) {
                    echo "No POSTINGS_LEN or LAST_BLOB_LEN, record postings".
                        " cannot be looked up\n";
                } else if ($is_postings_array) {
                    echo "Active Record\n-------------\n";
                    echo "By default list details of all postings\n";
                    var_dump($record['POSTINGS']);
                } else if ($details) {
                    $postings_entry = $index->getPostingsString(
                        $record['PARTITION'], $postings_offset,
                        $postings_len);
                    $postings = $index->postings_tools->unpack($postings_entry);
                    $index->deDeltaPostingsSumFrequencies($postings);
                    var_dump($postings);
                }
            }
            $i++;
            if ($num_records > 0 && $i >= $start_record + $num_records) {
                break;
            }
        }
    }
    /**
     * Prints information about the number of words and frequencies of words
     * within the $index'th partition in the bundle
     *
     * @param string $archive_path the path of a directory that holds
     *     an IndexDocumentBundle
     * @param int $num of partition to show info for
     */
    public function outputPartitionInfo($archive_path, $num)
    {
        if (preg_match("/\-\d$/", $archive_path)) {
            $bundle_num = substr($archive_path, -1);
            $archive_path = substr($archive_path, 0, -2);
        }
        $bundle_name = $this->getArchiveName($archive_path);
        echo "\nBundle Name: $bundle_name\n";
        $archive_type = $this->getArchiveKind($archive_path);
        echo "Bundle Type: $archive_type\n";
        if (!in_array($archive_type, ["FeedArchiveBundle",
            "DoubleIndexBundle", "IndexDocumentBundle",])) {
            $this->badFormatMessageAndExit($archive_path, "index");
        }
        preg_match("/\d+$/", $archive_path, $matches);
        $index_timestamp = (isset($matches[0])) ? $matches[0] : 0;
        if (isset($bundle_num) && $bundle_num >= 0) {
            $index_timestamp .= "-$bundle_num";
        } else if ($bundle_name == "IndexDataFeed") {
            $index_timestamp = "feed";
        }
        $index = IndexManager::getIndex($index_timestamp);
        $partition_bundle = $index->documents;
        $num_partitions = $partition_bundle->parameters["SAVE_PARTITION"] ?? -1;
        $num_partitions++;
        echo "Number of Partitions: $num_partitions\n";
        echo "\nInformation for Partition $num\n";
        echo "====================================\n";
        $_SERVER["NO_LOGGING"] = true;
        $partition = $partition_bundle->loadPartitionIndex($num);
        if (!$partition) {
            echo "Partition $num does not exists\n";
            return;
        }
        $statistics =
            $index->buildInvertedIndexPartition($num, null, true);
        echo "Number of Distinct Terms Indexed: " .
            count($statistics['TERM_STATISTICS'])."\n";
        echo "Number of Docs in Shard: " . $statistics['NUM_DOCS'] ."\n";
        echo "Number of Link Items in Shard: ".$statistics['NUM_LINKS']."\n";
        echo "Total Links and Docs: ".($statistics['NUM_DOCS'] +
            $statistics['NUM_LINKS'])."\n\n";
        echo "Term histogram for shard\n";
        echo "------------------------\n";
        $term_frequencies = $statistics['TERM_STATISTICS'];
        arsort($term_frequencies);
        $i = 1;
        echo "Freq Rank\t# Term\t# Docs Term Appears In\n";
        foreach ($term_frequencies as $term => $num_docs) {
            $term = rtrim($term, '_');
            $term = str_replace("3A", ":", $term);
            echo "$i\t\t\t$term\t\t\t$num_docs\n";
            $i ++;
        }
    }
    /**
     * Recomputes the hash index (.ix) files for a range of partitions
     * from start_partition to end_partition in the documents subfolder of
     * an IndexDocumentBundle. An ix file contains a sequence of compressed
     * 4-tuple (doc_id, summary_offset, summary_length, cache_length)
     * corresponding to a partition file (these end in .txt.gz and are
     * a sequence of compressed document summaries followed by orginal
     * documents).
     * @param string $archive_path the path of a directory that holds
     *     an IndexDocumentBundle
     * @param int $start_partition first partition to recompute
     * @param int $end_partition last partition to recompute (inclusive)
     */
    public function fixPartitionIndexes($archive_path, $start_partition,
        $end_partition = -1)
    {
        if (preg_match("/\-\d$/", $archive_path)) {
            $bundle_num = substr($archive_path, -1);
            $archive_path = substr($archive_path, 0, -2);
        }
        $bundle_name = $this->getArchiveName($archive_path);
        echo "\nBundle Name: $bundle_name\n";
        $archive_type = $this->getArchiveKind($archive_path);
        echo "Bundle Type: $archive_type\n";
        if (!in_array($archive_type, ["FeedArchiveBundle",
            "DoubleIndexBundle", "IndexDocumentBundle",])) {
            $this->badFormatMessageAndExit($archive_path, "index");
        }
        preg_match("/\d+$/", $archive_path, $matches);
        $index_timestamp = (isset($matches[0])) ? $matches[0] : 0;
        if (isset($bundle_num) && $bundle_num >= 0) {
            $index_timestamp .= "-$bundle_num";
        } else if ($bundle_name == "IndexDataFeed") {
            $index_timestamp = "feed";
        }
        $index = IndexManager::getIndex($index_timestamp);
        $partition_bundle = $index->documents;
        if ($end_partition == -1) {
            $end_partition = $partition_bundle->parameters["SAVE_PARTITION"];
        }
        $partition_tools = new PackedTableTools(
            ["PRIMARY KEY" => ["DOC_ID", IndexDocumentBundle::DOCID_LEN],
            "SUMMARY_OFFSET" => "INT", "SUMMARY_LENGTH" => "INT",
            "CACHE_PAGE_LENGTH" => "INT"]);
        $buffer_size = 1000000;
        $gz_start = '/\x1F\x8B\x08\x00\x00\x00\x00.../';
        for ($i = $start_partition; $i <= $end_partition; $i++) {
            $partition_filename = $partition_bundle->getPartition($i);
            echo "Reconstructing Index File (.ix) for $partition_filename\n";
            if (!file_exists($partition_filename)) {
                echo "\nPartition File: $partition_filename does not exists!";
                echo "\nStopping!";
                exit();
            }
            $first_time = true;
            $fh = fopen($partition_filename, "r");
            $remainder = "";
            $offset = 0;
            $is_summary = true;
            $cnt = 0;
            $partition_index = "";
            do {
                $gztext = fread($fh, $buffer_size);
                $objects = preg_split($gz_start, $remainder . $gztext);
                $num_objects = count($objects);
                $start = ($first_time) ? 1 : 0;
                $first_time = false;
                for ($j = $start ; $j < $num_objects; $j++) {
                    $compress = $objects[$j];
                    if ($j == $num_objects - 1 && !feof($fh)) {
                        $remainder = $compress;
                        break;
                    }
                    $len = 10 + strlen($compress);
                    $site_string = @gzinflate($compress);
                    if (empty($site_string)) {
                        echo "Couldn't uncompress item $cnt\n";
                        $offset += $len;
                        $last_summary = false;
                        continue;
                    }
                    $is_summary = (str_starts_with($site_string, 'a:')) ? true :
                        false;
                    $is_cache = (str_starts_with($site_string, 's:')) ? true :
                        false;
                    if ($is_summary) {
                        $site = @unserialize($site_string);
                        $last_summary = true;
                        if (empty($site)) {
                            echo "Couldn't unserialize item $cnt\n";
                            $offset += $len;
                            continue;
                        } else if (is_string($site)) {
                            echo "Item $cnt is a cache page not a summary.\n";
                            $offset += $len;
                            continue;
                        }
                        $doc_id = IndexDocumentBundle::computeDocId($site);
                        $summary_offset = $offset;
                        $summary_len = $len;
                    } else if ($is_cache && $last_summary) {
                        $last_summary = false;
                        $cache_len = $len;
                        if ($cnt % 1000 == 0) {
                            echo "...extracted and indexed $cnt items from:\n" .
                                "$partition_filename\n";
                            echo "Last Summary Offset: $summary_offset," .
                                " Summary Length: $summary_len, Cache Length: ".
                                "$cache_len\n";
                        }
                        $out_value = $partition_tools->pack([
                            "SUMMARY_OFFSET" => $summary_offset,
                            "SUMMARY_LENGTH" => $summary_len,
                            "CACHE_PAGE_LENGTH" => $cache_len
                        ]);
                        $partition_tools->add($partition_index, $doc_id,
                            $out_value, $partition_tools::ADD_MEM_TABLE_STRING);
                        $cnt++;
                    } else {
                        $last_summary = false;
                    }
                    $offset += $len;
                }
            } while(!feof($fh));
            $partition_index_name =
                $partition_bundle->getPartitionIndex($i);
            echo "Saving $partition_index_name.\n";
            $partition_tools->save($partition_index_name, $partition_index);
        }
    }
    /**
     * Counts and outputs the number of docs and links in each shard
     * in the archive supplied in $archive_path as well as an overall count
     *
     * @param string $archive_path patch of archive to count
     * @param bool $set_count flag that controls whether after computing
     *      the count to write it back into the archive
     */
    public function outputCountBundle($archive_path, $set_count = false)
    {
        $bundle_num = -1;
        if (preg_match("/\-\d$/", $archive_path)) {
            $bundle_num = substr($archive_path, -1);
            $archive_path = substr($archive_path, 0, -2);
        }
        $bundle_name = $this->getArchiveName($archive_path);
        echo "\nBundle Name: $bundle_name\n";
        $archive_type = $this->getArchiveKind($archive_path);
        echo "Bundle Type: $archive_type\n";
        if (!in_array($archive_type, ["FeedDocumentBundle",
            "DoubleIndexBundle", "IndexDocumentBundle",])) {
            $this->badFormatMessageAndExit($archive_path, "index");
        }
        preg_match("/\d+$/", $archive_path, $matches);
        $index_timestamp = (isset($matches[0])) ? $matches[0] : 0;
        if (isset($bundle_num) && $bundle_num >= 0) {
            $index_timestamp .= "-$bundle_num";
        } else if ($bundle_name == "IndexDataFeed") {
            $index_timestamp = "feed";
        }
        $index = IndexManager::getIndex($index_timestamp);
        $partition_bundle = $index->documents;
        $num_partitions = $partition_bundle->parameters["SAVE_PARTITION"] ?? -1;
        $num_partitions++;
        $partition_bundle->index_cache_size = 1;
        if ($num_partitions <= 0) {
            echo "Archive does not appear to have data yet";
            exit();
        } else {
            echo "Number of partitions: $num_partitions\n";
        }
        $count = 0;
        $visited_urls_count = 0;
        echo "Partition Counts\n===========\n";
        for ($i = 0; $i < $num_partitions; $i++ ) {
            $partition = $partition_bundle->loadPartitionIndex($i);
            $partition_keys = array_keys($partition);
            $docs_partition = 0;
            foreach($partition_keys as $partition_key) {
                if (IndexDocumentBundle::isType($partition_key, 'doc')) {
                    $docs_partition++;
                }
            }
            $num_links_and_docs =  count($partition_keys);
            echo "\nPartition:$i\n=======\n";
            echo "Number of Docs in Partition: " . $docs_partition . "\n";
            echo "Number of Link Items in Partition: " . $num_links_and_docs .
                "\n";
            $visited_urls_count += $docs_partition;
            $count += $num_links_and_docs;
        }
        echo "\n=======\n";
        echo "Total Number of Docs Seen:".$visited_urls_count."\n";
        echo "Total Number of Link or Doc Items:".$count."\n";
        if ($set_count == "save") {
            echo "\nSaving count to bundle...\n";
            $info = IndexDocumentBundle::getArchiveInfo($archive_path);
            $info['COUNT'] = $count - $docs_partition;
            $info['ACTIVE_COUNT'] = $docs_partition;
            $info['VISITED_URLS_COUNT'] = $visited_urls_count;
            IndexDocumentBundle::setArchiveInfo($archive_path, $info);
            echo "..done\n";
        }
    }
    /**
     * Given a complete path to an archive returns its filename
     *
     * @param string $archive_path a path to a yioop or non-yioop archive
     * @return string its filename
     */
    public function getArchiveName($archive_path)
    {
        $start = C\ARCHIVES_DIR . "/";
        if (strstr($archive_path, $start)) {
            $start_len = strlen($start);
            $name = substr($archive_path, $start_len);
        } else {
            $name = UrlParser::getDocumentFilename($archive_path);
        }
        return $name;
    }
    /**
     * Outputs to the terminal if the bloom filter $filter_path contains
     * the string $item
     * @param string $filter_path name of bloom filter file to check if
     *  contains item
     * @param string $item item to chheck in in bloom filter
     */
    public function checkFilter($filter_path, $item)
    {
        $item = trim($item);
        if (!file_exists($filter_path)) {
            echo "Filter File: $filter_path does not exist.";
            exit();
        }
        $filter = BloomFilterFile::load($filter_path);
        if ($filter->contains($item)) {
            echo "$item is contained in the filter\n";
        } else {
            echo "$item is not contained in the filter\n";
        }
    }
    /**
     * Makes a BloomFilterFile object from a dictionary file $dict_file which
     * has  items listed one per line, or items listed as some column of a CSV
     * file. The result is output to $filter_path
     * @param string $dict_file to make BloomFilterFile from
     * @param string $filter_path of file to serialize BloomFilterFile to
     * @param int $column_num if negative assumes $dict_file has one entry
     *  per line, if >=0 then is the index of the column in a csv to use
     *  for items
     */
    public function makeFilter($dict_file, $filter_path, $column_num = -1)
    {
        $lines = file($dict_file);
        $filter = new BloomFilterFile($filter_path, count($lines));
        $i = 0;
        foreach ($lines as $line) {
            $item = $line;
            if ($column_num != -1) {
                $line_parts = explode(",", $line);
                $item = $line_parts[$column_num] ?? "";
            }
            $item = trim($item, " \t\n\r\0\x0B\"\'");
            if (!empty($item)) {
                $item = mb_strtolower($item);
                $i++;
                if ($i % 10000 == 0) {
                    echo "Added $i items so far. Most recent: $item \n";
                }
                $filter->add($item);
            }
        }
        $filter->save();
    }
    /**
     * Outputs to stdout header information for a FeedDocumentBundle
     * bundle.
     *
     * @param array $info header info that has already been read from
     *     the description.txt file
     * @param string $archive_path file path of the folder containing the bundle
     * @param string $alternate_description used as the text for description
     *      rather than what's given in $info
     * @param bool $only_storage_info output only info about storage statistics
     *      don't output info about crawl parameters
     * @param bool $only_crawl_params output only info about crawl parameters
     *      not storage statistics
     */
    public function outputInfoFeedDocumentBundle($info, $archive_path,
        $alternate_description = "", $only_storage_info = false,
        $only_crawl_params = false)
    {
        $this->outputInfoIndexDocumentBundle($info, $archive_path,
            $alternate_description, $only_storage_info,
            $only_crawl_params);
    }
    /**
     * Outputs to stdout header information for a IndexDocumentBundle
     * bundle.
     *
     * @param array $info header info that has already been read from
     *     the description.txt file
     * @param string $archive_path file path of the folder containing the bundle
     * @param string $alternate_description used as the text for description
     *      rather than what's given in $info
     * @param bool $only_storage_info output only info about storage statistics
     *      don't output info about crawl parameters
     * @param bool $only_crawl_params output only info about crawl parameters
     *      not storage statistics
     */
    public function outputInfoIndexDocumentBundle($info, $archive_path,
        $alternate_description = "", $only_storage_info = false,
        $only_crawl_params = false)
    {
        $more_info = unserialize($info['DESCRIPTION']);
        $more_info = is_array($more_info) ? $more_info : [];
        unset($info['DESCRIPTION']);
        $info = array_merge($info, $more_info);
        $description = ($alternate_description) ? $alternate_description :
            "Description: " . $info['DESCRIPTION'];
        echo "$description\n";
        if (!$only_crawl_params) {
            $num_partitions = $info['SAVE_PARTITION'] + 1;
            echo "Number of partitions: $num_partitions \n";
            echo "Maximum number of documents or links per partition:" .
                $info['MAX_ITEMS_PER_FILE'] . "\n";
            echo "Number of stored links and documents: " .
                ($info['ACTIVE_COUNT'] + $info['COUNT']) .
                "\n";
            echo "Number of stored documents: " . $info['VISITED_URLS_COUNT'] .
                "\n";
        }
        if ($only_storage_info) {
            return;
        }
        if (isset($info['active_archive'])) {
            echo "Active Archive Bundle: " . $info['active_archive'] . "\n";
        }
        if (!empty($info['repeat_time'])) {
            echo "Last Swap Time: " . date("r", $info['repeat_time'])  . "\n";
        }
        if (!empty($info['repeat_frequency'])) {
            echo "Repeat Frequency: " . $info['repeat_frequency']  .
                " seconds\n";
        }
        $crawl_order = (isset($info[self::CRAWL_ORDER]) &&
            $info[self::CRAWL_ORDER] == self::BREADTH_FIRST) ?
            "Breadth First" : "Page Importance";
        echo "Crawl order was: $crawl_order\n";
        $channel = (isset($info[self::CHANNEL])) ? $info[self::CHANNEL] : 0;
        echo "Crawl Channel was: $channel.\n";
        if ($info['DESCRIPTION'] == 'feed') {
            echo "Feed Bundle, look at SearchSsources in web interface to see";
            echo "\n feed sources.\n";
        } else {
            echo "Seed sites:\n";
            foreach ($info[self::TO_CRAWL] as $seed) {
                echo "   $seed\n";
            }
            if ($info[self::RESTRICT_SITES_BY_URL]) {
                echo "Sites allowed to crawl:\n";
                foreach ($info[self::ALLOWED_SITES] as $site) {
                    echo "   $site\n";
                }
            }
            echo "Sites not allowed to be crawled:\n";
            if (is_array($info[self::DISALLOWED_SITES])) {
                foreach ($info[self::DISALLOWED_SITES] as $site) {
                    echo "   $site\n";
                }
            }
            echo "Page Rules:\n";
            if (isset($info[self::PAGE_RULES])) {
                foreach ($info[self::PAGE_RULES] as $rule) {
                    echo "   $rule\n";
                }
            }
            echo "\n";
        }
    }
    /**
     * Outputs to stdout header information for a DoubleIndexBundle
     * bundle.
     *
     * @param array $info header info that has already been read from
     *     the description.txt file
     * @param string $archive_path file path of the folder containing the bundle
     */
    public function outputInfoDoubleIndexBundle($info, $archive_path)
    {
        $this->outputInfoIndexArchiveBundle($info, $archive_path, "",
            false, true);
        $this->outputInfoIndexArchiveBundle($info, $archive_path . "/bundle0",
            "Bundle 0\n=======", true);
        echo "\n";
        $this->outputInfoIndexArchiveBundle($info, $archive_path . "/bundle1",
            "Bundle 1\n=======", true);
    }
    /**
     * Adds a list of urls as a upcoming schedule for a given queue bundle.
     * Can be used to make a closed schedule startable
     *
     * @param string $timestamp for a queue bundle to add urls to
     * @param string $url_file_name name of file consist of urls to inject into
     *      the given crawl
     */
    public function inject($timestamp, $url_file_name)
    {
        $admin = new AdminController();
        $machine_urls = $admin->model("machine")->getQueueServerUrls();
        $num_machines = count($machine_urls);
        $new_urls = file_get_contents($url_file_name);
        $inject_urls = $admin->convertStringCleanArray($new_urls);
        if (!$inject_urls || count($inject_urls) == 0) {
            echo "\nNo urls in $url_file_name to inject.\n\n";
            exit();
        }
        $crawl_model = $admin->model("crawl");
        $seed_info = $crawl_model->getCrawlSeedInfo($timestamp, $machine_urls);
        if (!$seed_info) {
            echo "\nNo queue bundle with timestamp: $timestamp.\n\n";
            exit();
        }
        $seed_info['seed_sites']['url'][] = "#\n#". date('r')."\n#";
        $seed_info['seed_sites']['url'] = array_merge(
            $seed_info['seed_sites']['url'], $inject_urls);
        $crawl_model->setCrawlSeedInfo($timestamp, $seed_info, $machine_urls);
        $crawl_model->injectUrlsCurrentCrawl($timestamp, $inject_urls,
            $machine_urls);
        echo "Urls injected!";
    }
    /**
     * Used to list out the pages/summaries stored in a bundle at
     * $archive_path. It lists to stdout $num many documents starting at $start.
     *
     * @param string $archive_path path to bundle to list documents for
     * @param int $start first document to list
     * @param int $num number of documents to list
     */
    public function outputShowPages($archive_path, $start, $num)
    {
        if (preg_match("/\-\d$/", $archive_path)) {
            $bundle_num = substr($archive_path, -1);
            $archive_path = substr($archive_path, 0, -2);
        }
        $fields_to_print = [
            self::URL => "URL",
            self::IP_ADDRESSES => "IP ADDRESSES",
            self::TIMESTAMP => "DATE",
            self::HTTP_CODE => "HTTP RESPONSE CODE",
            self::TYPE => "MIMETYPE",
            self::URL_PARENT => "URL OF PARENT",
            self::SCRAPER_LABEL => "SCRAPER LABEL",
            self::AVERAGE_COLOR => "AVERAGE_COLOR",
            self::DURATION => "DURATION",
            self::WIDTH => "WIDTH",
            self::HEIGHT => "HEIGHT",
            self::ENCODING => "CHARACTER ENCODING",
            self::DESCRIPTION => "DESCRIPTION",
            self::PAGE => "PAGE DATA"];
        $archive_type = $this->getArchiveKind($archive_path);
        if ($archive_type === false) {
            $this->badFormatMessageAndExit($archive_path);
        }
        $nonyioop = false;
        //for yioop archives we set up a dummy iterator
        $iterator =  (object) [];
        $iterator->end_of_iterator = false;
        $archive_name = C\NS_LIB . $archive_type;
        if ($archive_type == "IndexDocumentBundle" ||
            $archive_type == "FeedDocumentBundle") {
            $info = $archive_name::getArchiveInfo($archive_path);
            $num = min($num, $info["ACTIVE_COUNT"] + $info["COUNT"] - $start);
            $num_generations = $info['SAVE_PARTITION'] + 1;
            $index_archive = new IndexDocumentBundle($archive_path);
            $archive = $index_archive->documents;
        } else if ($archive_type == "DoubleIndexBundle") {
            $bundle_path = "$archive_path/bundle$bundle_num";
            $info = IndexDocumentBundle::getArchiveInfo($bundle_path);
            $num = min($num, $info["ACTIVE_COUNT"] + $info["COUNT"] - $start);
            $num_generations = $info['SAVE_PARTITION'] + 1;
            $index_archive = new IndexDocumentBundle($archive_path);
            $archive = $index_archive->documents;
        } else {
            $nonyioop = true;
            $num_generations = 1;
            //for non-yioop archives we set up a real iterator
            $iterator = $this->instantiateIterator($archive_path,
                $archive_type);
            if ($iterator === false) {
                $this->badFormatMessageAndExit($archive_path);
            }
        }
        if (!$nonyioop && isset($this->tmp_results)) {
                unset($this->tmp_results);
        }
        $num = max($num, 0);
        $total = $start + $num;
        $seen = 0;
        $generation = 0;
        while(!$iterator->end_of_iterator &&
            $seen < $total && $generation < $num_generations) {
            if ($nonyioop) {
                $partition = (object) [];
                $partition_count = 1;
                $iterator->seekPage($start);
                if ($iterator->end_of_iterator) {
                    break;
                }
                $seen += $start;
            } else {
                $partition = $archive->loadPartitionIndex($generation);
                $partition_count = count($partition);
                if ($seen + $partition_count < $start) {
                    $generation++;
                    $seen += $partition_count;
                    continue;
                }
                $keys = array_keys($partition);
                unset($partition);
                $seen_generation = 0;
            }
            $seen_generation = 0;
            while($seen < $total && $seen_generation < $partition_count) {
                if ($nonyioop) {
                    $num_to_get = min(self::MAX_BUFFER_DOCS, $total - $seen);
                    $objects = $iterator->nextPages($num_to_get);
                    $seen += count($objects);
                } else {
                    $num_to_get = min($total - $seen,
                        $partition_count - $seen_generation,
                        self::MAX_BUFFER_DOCS);
                    $objects = [];
                    for ($i = $seen_generation; $i < $num_to_get; $i++) {
                        $object = $archive->get($keys[$i], $generation);
                        $summary = $object[self::SUMMARY];
                        unset($object[self::SUMMARY]);
                        $object = array_merge($object, $summary);
                        $objects[] = $object;
                    }
                    $seen += $num_to_get;
                    $seen_generation += $num_to_get;
                }
                $num_to_get = count($objects);
                if ($seen >= $start) {
                    $num_to_show = min($seen - $start, $num_to_get);
                    $cnt = 0;
                    $first = $num_to_get - $num_to_show;
                    foreach ($objects as $object) {
                        if ($cnt >= $first) {
                            $out = "";
                            if (isset($object[self::TIMESTAMP])) {
                                $object[self::TIMESTAMP] =
                                    date("r", $object[self::TIMESTAMP]);
                            }
                            foreach ($fields_to_print as $key => $name) {
                                if (isset($object[$key])) {
                                    $out .= "[$name]\n";
                                    if ($key != self::IP_ADDRESSES) {
                                        $out .= $object[$key]."\n";
                                    } else {
                                        foreach ($object[$key] as $address) {
                                            $out .= $address."\n";
                                        }
                                    }
                                }
                            }
                            $out .= "==========\n\n";
                            echo "BEGIN ITEM, LENGTH:".strlen($out)."\n";
                            echo $out;
                        }
                        $cnt++;
                    }
                }
                if ($objects == null) break;
            }
            $generation++;
        }
        if (isset($this->tmp_results)) {
            //garbage collect savepoint folder for non-yioop archives
            $dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
            $db = new $dbms_manager();
            $db->unlinkRecursive($this->tmp_results);
        }
    }
    /**
     * Copies an IndexArchiveBundle (or derived class) on disk into
     * an IndexDocumentBundle (on disk).  The new bundle will be
     * at the old bundle's location while the old bundle is renamed
     * after the process to "Old" . name_of_old_bundle
     *
     * @param string $archive_path file path to a IndexArchiveBundle
     * @param int $number_of_processes number of processes that should be
     *  used to perform the migration (more is faster, but this is only used
     *  for migration of indexes between 3.0 and 4.0)
     */
    public function migrateIndexArchive($archive_path, $number_of_processes = 1)
    {
        $archive_type = $this->getArchiveKind($archive_path);
        if (in_array($archive_type, ["FeedDocumentBundle",
            "IndexDocumentBundle"])) {
            $version = IndexManager::getVersion($archive_path);
            if ($version < "4.0") {
                echo "Checking if bundle upgradable...\n";
                echo "--------------------------------\n";
                echo "...Feed or IndexDocumentBundle Version $version ".
                    "detected!\n";
                echo "...Upgrading bundle to version ".
                    IndexDocumentBundle::DEFAULT_VERSION . "...\n";
                $this->migrateIndexDocument($archive_path,
                    $number_of_processes);
                return;
            }
        }
        $archive_type_use = ($archive_type == "DoubleIndexBundleOld") ?
            "DoubleIndexBundle" : $archive_type;
        $archive_name = C\NS_LIB . $archive_type_use;
        $archive_map = [
            "DoubleIndexBundleOld" => "DoubleIndexBundle",
            "FeedArchiveBundle" => "FeedDocumentBundle",
            "IndexArchiveBundle" => "IndexDocumentBundle",
        ];
        if (!isset($archive_map[$archive_type])) {
            $this->badFormatMessageAndExit($archive_path, "migratable index");
        }
        $file_name_pos = strrpos($archive_path, "/");
        $parent_path = substr($archive_path, 0, $file_name_pos);
        $file_name = substr($archive_path, $file_name_pos + 1);
        $out_archive_path = "$parent_path/Tmp$file_name";
        $new_archive_type = $archive_map[$archive_type];
        $new_archive_name = C\NS_LIB . $new_archive_type;
        $info = $archive_name::getArchiveInfo($archive_path);
        if ($new_archive_type == "FeedDocumentBundle") {
            $dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
            $db = new $dbms_manager();
            $db->connect();
            $out_index_archive = new $new_archive_name($out_archive_path, $db,
                false, $info["DESCRIPTION"], C\NUM_DOCS_PER_PARTITION);
            $sql = "SELECT * FROM MEDIA_SOURCE WHERE (TYPE='rss'
                 OR TYPE='html' OR TYPE='json' OR TYPE='regex')";
            $result = $db->execute($sql);
            $feeds = [];
            while ($feed = $db->fetchArray($result)) {
                FeedsUpdateJob::parseFeedAuxInfo($feed);
                $feeds[] = $feed;
            }
            $out_index_archive->feeds = $feeds;
        } else if ($new_archive_type == "DoubleIndexBundle") {
            $out_index_archive = new $new_archive_name($out_archive_path,
                false, $info["DESCRIPTION"], C\NUM_DOCS_PER_PARTITION,
                $info["repeat_frequency"]);
        } else {
            $out_index_archive = new $new_archive_name($out_archive_path,
                false, $info["DESCRIPTION"], C\NUM_DOCS_PER_PARTITION);
        }
        $num_bundles = ($new_archive_type == "DoubleIndexBundle") ? 2 : 1;
        for ($i = 0; $i < $num_bundles; $i++) {
            $bundle_path = ($new_archive_type == "DoubleIndexBundle") ?
                "$archive_path/bundle$i" : $archive_path;
            $archive = new WebArchiveBundle($bundle_path . "/summaries");
            L\crawlLog("Processing bundle $i.");
            $generation_info = unserialize(
                file_get_contents("$bundle_path/generation.txt"));
            $num_generations = $generation_info['ACTIVE'] + 1;
            $seen = 0;
            $generation = 0;
            $keypad = "\x00\x00\x00\x00";
            while($generation < $num_generations) {
                $partition = $archive->getPartition($generation, false);
                L\crawlLog("Processing partition $generation");
                L\crawlLog("Number of objects in partition:" .
                    $partition->count);
                L\crawlLog("Partition Version:" . $partition->version);
                $seen_partition = 0;
                $more_objects = ($partition->count > 0);
                while($more_objects) {
                    $num_to_get = self::MAX_REBUILD_DOCS;
                    $offset = $partition->iterator_pos;
                    $objects = $partition->nextObjects($num_to_get);
                    if (empty($objects)) {
                        $more_objects = false;
                        continue;
                    }
                    $seen_partition += count($objects);
                    $cnt = 0;
                    $store_sites = [];
                    foreach ($objects as $object) {
                        $site = $object[1];
                        if (isset($site['DUMMY_OFFSET']) || empty($site)) {
                            // first item in a partition is a dummy record
                            continue;
                        }
                        if (!isset($site[self::TYPE]) ||
                            $site[self::TYPE] != "link") {
                            $cnt++;
                        }
                        $doc_id = $out_index_archive->computeDocId($site);
                        if ($doc_id) {
                            $page = $site[self::PAGE] ?? "";
                            unset($site[self::PAGE]);
                            $store_sites[] = [self::DOC_ID => $doc_id,
                                self::SUMMARY => $site, self::PAGE => $page];
                        }
                    }
                    $seen_partition += $num_to_get;
                    unset($objects);
                    if (!empty($store_sites)) {
                        if ($new_archive_type == "FeedDocumentBundle") {
                            $result = $out_index_archive->addPagesAndSeenKeys(
                                $store_sites, $cnt);
                        } else {
                            $result = $out_index_archive->addPages($store_sites,
                                $cnt);
                        }
                        if ($result) {
                            L\crawlLog("..Store $cnt pages succeeded.");
                        } else {
                            L\crawlLog("..STORE ERROR!!!! for the following ".
                                "documents:");
                            L\crawlLog(print_r($seen_sites, true));
                            L\crawlLog("..Store $cnt pages failed.");
                        }
                        unset($store_sites);
                    }
                }
                $out_index_archive->updateDictionary();
                $generation++;
            }
            if ($new_archive_type == "DoubleIndexBundle" &&
                $i < 1) {
                $out_index_archive->swapActiveBundle();
            } else {
                $out_index_archive->forceSave();
            }
        }
        if ($new_archive_type == "DoubleIndexBundle") {
            $status = ["repeat_frequency" => $info['repeat_frequency'],
                "repeat_time" => $info['repeat_time'],
                "swap_count" => $info['swap_count'],
                "DESCRIPTION" => $out_index_archive->description
            ];
            file_put_contents($out_index_archive->dir_name . "/status.txt",
                serialize($status));
            copy($archive_path . "/StartCrawlSchedule.txt",
                $out_index_archive->dir_name . "/StartCrawlSchedule.txt");
        }
        $old_archive_path = "$parent_path/Old$file_name";
        rename($archive_path, $old_archive_path);
        rename($out_archive_path, $archive_path);
        echo <<< EOD

Migration complete!

Test out the migrated index to see if it is working correctly.
The original index has been stored at:
$old_archive_path
If the new migrated index is working correctly you can delete the original
index. Otherwise, you can move the original index back to
$archive_path
to use it again.

EOD;
    }
    /**
     * Migrates a Yioop >=3 index to the most recent index format of Yioop.
     *
     * @param string $archive_path path to a Yioop IndexDocumentBundle to
     *   migrate
     * @param int $number_of_processes number of computer processes to use
     *   to performt the migration (more should be faster)
     */
    public function migrateIndexDocument($archive_path, $number_of_processes=1)
    {
        $new_documents_folder = $archive_path . "/" .
            IndexDocumentBundle::DOCUMENTS_FOLDER;
        $old_documents_folder = "$archive_path/documents";
        if (!file_exists($old_documents_folder)) {
            echo "...Could not find any documents in bundle. Stopping!\n";
            return;
        }
        rename($old_documents_folder, $old_documents_folder . "-old");
        $old_documents_folder .= "-old";
        if (!file_exists($new_documents_folder)) {
            echo "...Creating new Documents folder\n";
            mkdir($new_documents_folder);
            chmod($new_documents_folder, 0777);
        }
        $old_param_file = "$old_documents_folder/pdb_parameters.txt";
        if (file_exists($old_param_file)) {
            $pdb_parameters = unserialize(file_get_contents($old_param_file));
            $pdb_parameters["OVERFLOW_THRESHOLD"] = C\OVERFLOW_THRESHOLD;
            $pdb_parameters["PARTITION_SIZE_THRESHOLD"] =
                PartitionDocumentBundle::PARTITION_SIZE_THRESHOLD;
            file_put_contents("$new_documents_folder/pdb_parameters.txt",
                serialize($pdb_parameters));
        }
        $old_partitions = glob("$old_documents_folder/partition_*.txt.gz");
        $i = 0;
        $folder_path = $new_documents_folder;
        foreach ($old_partitions as $old_partition) {
            $block = sprintf("%'.05d",
                floor($i / PartitionDocumentBundle::BLOCK_FACTOR));;
            if (C\OVERFLOW_THRESHOLD > 0 &&
                $old_partition > C\OVERFLOW_THRESHOLD) {
                $folder_path .=
                    "/" . PartitionDocumentBundle::OVERFLOW_DIR_FOLDER;
                if (!file_exists($folder_path)) {
                    mkdir($folder_path);
                    chmod($folder_path, 0777);
                }
            }
            $block_path = "$new_documents_folder/".
                PartitionDocumentBundle::BLOCK_PREFIX . $block;
            if (!file_exists($block_path)) {
                mkdir($block_path);
                chmod($block_path, 0777);
            }
            $partition_path = "$block_path/".
                PartitionDocumentBundle::PARTITION_PREFIX .
                sprintf("%'.010d", $i);
            if (!file_exists($partition_path)) {
                mkdir($partition_path);
                chmod($partition_path, 0777);
            }
            $dir_name =  pathinfo($old_partition, PATHINFO_DIRNAME);
            $base_name =  pathinfo($old_partition, PATHINFO_BASENAME);
            $file_path = substr($old_partition, 0, -strlen(".txt.gz"));
            rename($old_partition, "$partition_path/" .
                PartitionDocumentBundle::ARCHIVE_FILENAME . ".txt.gz");
            if (file_exists("$file_path.ix")) {
                rename("$file_path.ix", "$partition_path/" .
                    PartitionDocumentBundle::INDEX_FILENAME);
            }
            $i++;
        }
        $dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
        $db = new $dbms_manager();
        $db->unlinkRecursive($archive_path . "/" .
            IndexDocumentBundle::DICTIONARY_FOLDER, true);
        $db->unlinkRecursive($archive_path . "/positions_doc_maps", true);
        $db->unlinkRecursive($old_documents_folder);
        echo "...Building Dictionary in new format:\n";
        $this->rebuildIndexBundle($archive_path, 0, true, $number_of_processes);
        echo "...Dictionary Rebuilt!\n";
        $archive_info = unserialize(file_get_contents($archive_path . "/" .
            IndexDocumentBundle::ARCHIVE_INFO_FILE));
        $archive_info["VERSION"] = IndexDocumentBundle::DEFAULT_VERSION;
        file_put_contents($archive_path . "/" .
            IndexDocumentBundle::ARCHIVE_INFO_FILE, serialize($archive_info));
        echo "...Updating bundles version number!\n";
        echo "Bundle Upgrade Complete!\n";
    }
    /**
     * Used to recompute both the index shards and the dictionary
     * of an index archive. The first step involves re-extracting the
     * word into an inverted index from the summaries' web_archives.
     * Then a reindex is done.
     *
     * @param string $archive_path file path to a IndexArchiveBundle
     * @param mixed $start_generation which web archive generation to start
     *  rebuild from. If 'continue' then keeps going from where last attempt at
     *  a rebuild was.
     * @param bool $force_recompute whether to force the recomputation of
     *   partition inverted indexes or to try to use existing ones if present
     * @param int $number_of_processes number of CPU processes to use
     *   when trying to recompute partition inverted indexes
     * @param mixed $last_generation which web archive generation to end
     *  rebuild at. Default value of -1 goes till last partition
     */
    public function rebuildIndexBundle($archive_path, $start_generation = 0,
        $force_recompute = true, $number_of_processes = 1,
        $last_generation = -1)
    {
        $rebuilding = ($force_recompute) ? "Rebuilding" : "Remerging";
        $rebuild = ($force_recompute) ? "rebuild" : "remerge";
        $sleep_time = ($force_recompute) ? 15 : 3;
        if (!$force_recompute) {
            $number_of_processes = 1;
        }
        L\crawlLog("$rebuilding index using $number_of_processes processes!!");
        $bundle_num = -1;
        $bundle_path = $archive_path;
        if (preg_match("/\-\d$/", $archive_path)) {
            $bundle_num = substr($archive_path, -1);
            $archive_path = substr($archive_path, 0, -2);
            /* we can rebuild DoubleIndexBundle's by rebuilding each archive
               separate
             */
            $bundle_path = "$archive_path/bundle$bundle_num";
        }
        $archive_type = $this->getArchiveKind($bundle_path);
        $archive_name = C\NS_LIB . $archive_type ;
        if (!in_array($archive_type, ["FeedDocumentBundle",
            "IndexDocumentBundle"])) {
            $this->badFormatMessageAndExit($archive_path, "rebuildable index");
        }
        $next_partition_path = $bundle_path . "/".
            IndexDocumentBundle::NEXT_PARTITION_FILE;
        if (trim($start_generation) === "continue") {
            if (file_exists($next_partition_path)) {
                $start_generation =
                    intval(file_get_contents($next_partition_path));
                echo "Restarting $rebuild index from $start_generation\n";
            } else {
                $start_generation = 0;
            }
        }
        $dictionary_path = $bundle_path .
            IndexDocumentBundle::DICTIONARY_FOLDER;
        $dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
        $db = new $dbms_manager();
        if ($start_generation == 0) {
            $db->unlinkRecursive($bundle_path . "/" .
                IndexDocumentBundle::DICTIONARY_FOLDER, false);
        }
        file_put_contents($next_partition_path, $start_generation);
        if ($archive_type == "FeedDocumentBundle") {
            $dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
            $db = new $dbms_manager();
            $db->connect();
            $index_archive = new $archive_name($bundle_path, $db,
                false, null, C\NUM_DOCS_PER_PARTITION);
            $sql = "SELECT * FROM MEDIA_SOURCE WHERE (TYPE='rss'
                 OR TYPE='html' OR TYPE='json' OR TYPE='regex')";
            $result = $db->execute($sql);
            $feeds = [];
            while ($feed = $db->fetchArray($result)) {
                FeedsUpdateJob::parseFeedAuxInfo($feed);
                $feeds[] = $feed;
            }
            $index_archive->feeds = $feeds;
        } else {
            $index_archive = new $archive_name($bundle_path,
                false, "", C\NUM_DOCS_PER_PARTITION);
        }
        $save_partition = $index_archive->documents->parameters[
            "SAVE_PARTITION"] ?? 0;
        L\crawlLog("Save partition is $save_partition");
        $updater_option =  ($force_recompute) ? "rebuild_%d" : "remerge";
        $options = "run %d '$bundle_path' $updater_option";
        $old_next_partition = -1;
        $next_partition = $start_generation;
        $continue = false;
        $recent_log_times = [];
        $dictionary_logs = [];
        for ($i = 0; $i < $number_of_processes; $i++) {
            $dictionary_logs[$i] = C\LOG_DIR . "/$i-DictionaryUpdater.log";
            $fp = fopen($dictionary_logs[$i], "w");
            fclose($fp);
            $recent_log_times[$i] = time();
        }
        $rebuild_dones = [];
        $end_partition = ($last_generation > 0) ? min($save_partition,
            $last_generation) : $save_partition;
        while ($next_partition < $end_partition) {
            if ($old_next_partition != $next_partition) {
                $old_next_partition = $next_partition;
                $num_forks = min($end_partition - $next_partition,
                    $number_of_processes);
                echo "Num forks: $num_forks, ".
                    "num processes $number_of_processes\n";
                for ($i = 0; $i < $num_forks; $i++) {
                    $process_partition = $next_partition + $i;
                    $process_options = sprintf($options, $i,
                        $process_partition);
                    L\crawlLog(
                        "Begin Processing Partition: $process_partition");
                    L\crawlLog("Exec'ing DictionaryUpdater with parameters: " .
                        $process_options);
                    L\CrawlDaemon::execScriptInOwnProcess(C\BASE_DIR .
                        "/executables/DictionaryUpdater.php", $process_options);
                }
            } else {
                for ($i = 0; $i < $num_forks; $i++) {
                    if (file_exists($dictionary_logs[$i])) {
                        $recent_log_data =
                            file_get_contents($dictionary_logs[$i]);
                        if (!empty($recent_log_data)) {
                            $recent_log_times[$i] = time();
                            echo "Process $i:\n----------\n";
                            echo $recent_log_data;
                        } else {
                            echo "Waiting on data from process $i\n";
                        }
                        $fp = fopen($dictionary_logs[$i], "w");
                        fclose($fp);
                    }
                }
                clearstatcache();
                for ($i = 0; $i < $num_forks; $i++) {
                    if (empty($rebuild_dones[$i]) &&
                        time() > $recent_log_times[$i] + 5 * C\LOG_TIMEOUT) {
                        L\crawlLog("DictionaryUpdater seems to have crashed, ".
                            "exiting ArcTool");
                        exit();
                    }
                }
                sleep(5);
            }
            $finished = true;
            for ($i = 0; $i < $num_forks; $i++) {
                $rebuild_status_file = "$bundle_path/rebuild_$i";
                if (file_exists($rebuild_status_file)) {
                    $rebuild_dones[$i] = true;
                } else {
                    $finished = false;
                }
            }
            if ($finished) {
                $next_partition += $num_forks;
                file_put_contents($next_partition_path, $next_partition);
                $rebuild_dones = [];
                for ($i = 0; $i < $num_forks; $i++) {
                    $rebuild_status_file = "$bundle_path/rebuild_$i";
                    unlink($rebuild_status_file);
                    $recent_log_times[$i] = time();
                }
            }
        }
        if ($force_recompute) {
            /* when force recompute, we first build inverted indexes for
              all partition, then the call below is used to remerge them into
              the global dictionary
              */
            $this->rebuildIndexBundle($archive_path, $start_generation, false,
                last_generation: $last_generation);
        }
        echo "\nIndex $rebuild complete!\n";
    }
    /**
     * Used to create an archive_bundle_iterator for a non-yioop archive
     * As these iterators sometimes make use of a folder to store savepoints
     * We create a temporary folder for this purpose in the current directory
     * This should be garbage collected elsewhere.
     *
     * @param string $archive_path path to non-yioop archive
     * @param string $iterator_type name of archive_bundle_iterator used to
     *     iterate over archive.
     * @param return an ArchiveBundleIterator of the correct type using
     *     a temporary folder to store savepoints
     */
    public function instantiateIterator($archive_path, $iterator_type)
    {
        $iterate_timestamp = filectime($archive_path);
        $result_timestamp = strval(time());
        $this->tmp_results = C\TEMP_DIR . '/TmpArchiveExtract'.
            $iterate_timestamp;
        $dbms_manager = C\NS_DATASOURCES. ucfirst(C\p('DBMS')) . "Manager";
        $db = new $dbms_manager();
        if (file_exists($this->tmp_results)) {
            $db->unlinkRecursive($this->tmp_results);
        }
        @mkdir($this->tmp_results);
        $iterator_class = C\NS_ARCHIVE . "{$iterator_type}Iterator";
        $iterator = new $iterator_class($iterate_timestamp, $archive_path,
            $result_timestamp, $this->tmp_results);
        $db->setWorldPermissionsRecursive($this->tmp_results);
        return $iterator;
    }
    /**
     * Retrieves the log data from the ImpressionRecords folder.
     *
     * @param string $key Optional primary key (user_id) to filter records by.
     * @param int $partition Optional partition number; defaults to the
     *      current SAVE_PARTITION.
     * @param int $limit Optional maximum number of records to display;
     *      defaults to all records.
     * @return void
     */
    public function displayLogRecords($key = null, $partition = null,
        $limit = null)
    {
        $folder = C\LOG_DIR . "/" . ImpressionModel::IMPRESSION_FOLDER;
        $format = ImpressionModel::IMPRESSION_FORMAT;
        $pdb = new PartitionDocumentBundle($folder, $format);
        $max_partition = $pdb->parameters['SAVE_PARTITION'];
        $partitions = ($partition === null) ? range(0, $max_partition) :
            [ (int)$partition ];
        foreach ($partitions as $part) {
            $index = $pdb->loadPartitionIndex($part, false,
                PackedTableTools::APPEND_MODE);
            $keys_to_process = ($key === null) ? array_keys($index) :
                [ (string)$key ];
            foreach ($keys_to_process as $user_id) {
                if (! isset($index[$user_id])) {
                    continue;
                }
                $packed_row = $index[$user_id];
                $rows = ($limit === null) ?
                    $pdb->table_tools->unpack($packed_row)
                    : $pdb->table_tools->unpack($packed_row, 0, $limit);
                foreach ($rows as $record) {
                    echo "Key: {$user_id}\n";
                    print_r($record);
                    echo "-------------------------\n";
                }
            }
        }
    }
    /**
     * Like displayLogRecords(), but filters by item_id instead of user_id.
     *
     * @param int $item_id    The item ID to filter on (required)
     * @param int $partition Optional partition index
     * @param int $limit     Optional max number of records
     */
    public function displayLogRecordsByItemId($item_id, $partition = null,
       $limit = null)
    {
        if (!is_numeric($item_id)) {
            echo "Invalid item_id\n";
            return;
        }
        $folder = C\LOG_DIR . "/" . ImpressionModel::IMPRESSION_FOLDER;
        $format = ImpressionModel::IMPRESSION_FORMAT;
        $pdb = new PartitionDocumentBundle($folder, $format);
        $max_partition = $pdb->parameters['SAVE_PARTITION'];
        $partitions = ($partition !== null) ? [(int)$partition] :
            range(0, $max_partition);
        $matched = [];
        foreach ($partitions as $part) {
            $index = $pdb->loadPartitionIndex($part, false,
               PackedTableTools::APPEND_MODE);
            if (empty($index)) {
                continue;
            }
            foreach ($index as $key => $packed_row) {
                $rows = $pdb->table_tools->unpack($packed_row);
                foreach ($rows as $record) {
                    if ($record['item_id'] == $item_id) {
                        $matched[] = ['key' => $key, 'record' => $record];
                        if ($limit !== null && count($matched) >= $limit) {
                            break 3;
                        }
                    }
                }
            }
        }
        if (empty($matched)) {
            echo "No records found for item_id: $item_id\n";
            return;
        }
        foreach ($matched as $entry) {
            echo "Key: {$entry['key']}\n";
            print_r($entry['record']);
            echo "-------------------------\n";
        }
    }
    /**
     * Retrieve and display all item IDs associated with a given user.
     *
     * This method queries the ITEM_IMPRESSION_SUMMARY table for entries
     * matching the specified USER_ID. Each matching record’s USER_ID column is
     * printed on its own line to stdout.
     *
     * @param int $user_id The user identifier to filter GROUP_ITEM records by.
     */
    public function getItemIdByUser(int $user_id)
    {
        $model = new ImpressionModel();
        $db = $model->db;
        $dbinfo = ["DBMS" => C\p('DBMS'), "DB_HOST" => C\p('DB_HOST'),
            "DB_USER" => C\p('DB_USER'), "DB_PASSWORD" => C\p('DB_PASSWORD'),
            "DB_NAME" => C\p('DB_NAME') ];
        $sql = "SELECT ITEM_ID FROM ITEM_IMPRESSION_SUMMARY WHERE USER_ID = ?";
        $res = $db->execute($sql, [$user_id]);
        while ($row = $db->fetchArray($res)) {
            echo $row['ITEM_ID'], "\n";
        }
    }
    /**
     * Retrieve and display all item IDs belonging to a specific group.
     *
     * This method queries the GROUP_ITEM table for entries matching the
     * specified GROUP_ID. Each resulting record’s ID column is printed on its
     * own line.
     *
     * @param int $group_id The group identifier to filter GROUP_ITEM records
     *                      by.
     */
    public function getItemIdByGroup(int $group_id)
    {
        $model = new ImpressionModel();
        $db = $model->db;
        $dbinfo = ["DBMS" => C\p('DBMS'), "DB_HOST" => C\p('DB_HOST'),
            "DB_USER" => C\p('DB_USER'), "DB_PASSWORD" => C\p('DB_PASSWORD'),
            "DB_NAME" => C\p('DB_NAME') ];
            $sql = "SELECT ID FROM GROUP_ITEM WHERE GROUP_ID = ?";
        $res = $db->execute($sql, [$group_id]);
        while ($row = $db->fetchArray($res)) {
            echo $row['ID'], "\n";
        }
    }
    /**
     * Retrieve and display all item IDs of a specific type.
     *
     * Accepts either a human-readable type name (thread, wiki, group,
     * query, cache, resource) or the corresponding integer code. Looks
     * up the integer code via an internal map, then queries
     * ITEM_IMPRESSION_SUMMARY for rows matching that type and prints each
     * ITEM_ID on its own line. If no records match, outputs
     * "No Items of that type".
     *
     * @param string|int $item_type The type name or numeric code to use.
     * Valid names: thread, wiki, group, query, cache, resource.
     * Numeric codes: 1 (thread), 2 (wiki), 3 (group), 4 (query), 5 (cache),
     * 6 (resource).
     */
    public function getItemIdByType($item_type)
    {
        $type_map = ['thread' => C\THREAD_IMPRESSION,
            'wiki' => C\WIKI_IMPRESSION, 'group' => C\GROUP_IMPRESSION,
            'query' => C\QUERY_IMPRESSION, 'cache' => C\CACHE_IMPRESSION,
            'resource' => C\RESOURCE_IMPRESSION];
        if (is_string($item_type)) {
            $key = strtolower($item_type);
            if (!isset($type_map[$key])) {
                $valid = implode(', ', array_keys($type_map));
                echo "Unknown item type '{$item_type}'. Valid types are: " .
                "{$valid}\n";
               return;
           }
            $type_id = $type_map[$key];
        } else {
            $type_id = (int)$item_type;
        }
        $model = new ImpressionModel();
        $db = $model->db;
        $dbinfo = ["DBMS" => C\p('DBMS'), "DB_HOST" => C\p('DB_HOST'),
            "DB_USER" => C\p('DB_USER'), "DB_PASSWORD" => C\p('DB_PASSWORD'),
            "DB_NAME" => C\p('DB_NAME')];
        $sql ="SELECT ITEM_ID FROM ITEM_IMPRESSION_SUMMARY WHERE ITEM_TYPE = ?";
        $res = $db->execute($sql, [$type_id]);
        $row = $db->fetchArray($res);
        if (!$row) {
            echo "No Items of that type\n";
        } else {
            do {
                echo $row['ITEM_ID'], "\n";
            } while ($row = $db->fetchArray($res));
        }
    }
    /**
     * Outputs the "hey, this isn't a known bundle message" and then exit()'s.
     *
     * @param string $archive_name name or path to what was supposed to be
     *     an archive
     * @param string $allowed_archives a string list of archives types
     *     that $archive_name could belong to
     */
    public function badFormatMessageAndExit($archive_name,
        $allowed_archives = "web or index")
    {
        echo <<< EOD

$archive_name does not appear to be a $allowed_archives archive bundle

EOD;
        exit();
    }

    /**
     * Outputs the "how to use this tool message" and then exit()'s.
     */
    public function usageMessageAndExit()
    {
        echo  <<< EOD

ArcTool is used to look at the contents of FeedDocumentBundle's,
IndexDocumentBundle's, DoubleIndexBundle's, and BloomFilterFile's. It will look
for these using the path provided or will check in the Yioop! crawl directory
as a fall back.

The available commands for ArcTool are:

php ArcTool.php check-filter filter_file item
    /* outputs whether item is in the BloomFilterFile given by filter_file
     */

php ArcTool.php count bundle_name
php ArcTool.php count double_index_name which_bundle
    or
php ArcTool.php count bundle_name save
php ArcTool.php count double_index_name which_bundle save
    /* returns the counts of docs and links for each partition in a bundle
       as well as an overall total. The second command saves the just
       computed count into the index description (can be used to
       fix the index count if it gets screwed up).
     */
php ArcTool.php doc-lookup bundle_name partition doc_map_index
    /*
       returns the document stored in partition at doc_map_index
       (here doc_map_index is the value that would be stored in a posting)
    */
php ArcTool.php dict bundle_info word [details]
php ArcTool.php dict bundle_info word start_record num_records [details]
    /* returns index dictionary records for word stored in index archive bundle
       or double index bundle. bundle_info is either the name of the bundle,
       or for double index bundle, the name of the bundle whitespace which
       sub-bundle. In the later case you should provide which bundle
       you want dictionary info for. This command also supports start
       and number of record parameters. If the word details is added to the end
       of the command then additional information about each
       doc record (as opposed to just their total number) is printed
     */

php ArcTool.php fix-partition bundle_name
php ArcTool.php fix-partition bundle_name start_partition
php ArcTool.php fix-partition bundle_name start_partition end_partition
    /*  recomputes the hash index (index) file in each partition folder
        (BSOME_NUMBER/PSOME_NUMBER folder)
        for a range of partitions from start_partition to end_partition in the
        Documents subfolder of an IndexDocumentBundle. An index file contains
        a sequence of compressed 4-tuple (doc_id, summary_offset,
        summary_length, cache_length)
        corresponding to a archive.txt.gz file (these  are
        a sequence of compressed document summaries followed by original
        documents).
     */

php ArcTool.php info bundle_name
    // return info about documents stored in archive.

php ArcTool.php inject timestamp file
    /* injects the urls in file as a schedule into crawl of given timestamp
        This can be used to make a closed index unclosed and to allow for
        continued crawling. */

php ArcTool.php list
    /* returns a list of all the archives in the Yioop! crawl directory,
       including non-Yioop! archives in the /archives sub-folder.*/

php ArcTool.php make-filter dict_file filter_file
php ArcTool.php make-filter dict_file filter_file column_num
    /* outputs to filter_file a BloomFilterFile made by inserting the items
       in dict_file. If column_num is negative then dict_file is assumed to
       list one item to insert per line. If column_num >=0 then dict_file
       is assumed to be a csv file and column_num is the column that items
       will be inserted from.
     */

php ArcTool.php migrate bundle_name
php ArcTool.php migrate bundle_name number_of_processes
    /* migrates old Yioop index formats such as FeedArchiveBundle,
       IndexArchiveBundle, and old style DoubleIndexBundle to their
       modern respective equivalents FeedDocumentBundle, IndexDocumentBundle,
       and modern DoubleIndexBundle. It can also migrate a version < 4
       FeedDocumentBundle, IndexDocumentBundle to a current version bundle
       The default number of processes that are used for index rebuilding is 1,
       however, rebuilding is faster if you specify a number_of_processes
       proportional to the number of CPU cores of your machine.
    */

php ArcTool.php partition bundle_name partition_number
php ArcTool.php partition double_index_name which_bundle partition_number
    /* Prints information about the number of words and frequencies of words
       within the partition_number'th partition in the index document bundle
       or double index bundle (in which case need to say either 0 or  1 bundle)
     */

php ArcTool.php rebuild bundle_info
php ArcTool.php rebuild bundle_info partition_num
php ArcTool.php rebuild bundle_info partition_num number_of_processes
php ArcTool.php rebuild bundle_info partition_num number_of_processes end_partition
    /*  re-extracts words from summaries files in bundle_name a partition at a
        time, builds an inverted index for that partition and adds to the global
        dictionary. bundle_info is either the name of the bundle, or for
        double index bundle, the name of the bundle whitespace which sub-bundle.
        If this process crashes the keyword continue can be used
        to continue from where it left off. If a partition_num is supplied
        process continue from that partition number. The keyword "continue" can
        be used to continue from the last processed partitition numebr.
        The default number of processes that are used for index rebuilding is 1,
        however, rebuilding is faster if you specify a number_of_processes
        proportional to the number of CPU cores of your machine.
        */

php ArcTool.php remerge bundle_info
php ArcTool.php remerge bundle_info partition_num
php ArcTool.php remerge bundle_info partition_num number_of_processes
php ArcTool.php remerge bundle_info partition_num number_of_processes end_partition
    /*  this operates like the previously described rebuild command except
        if the inverted index files for a partition already exist in that
        partition they are not recomputed (if they don't exist, they
        are recomputed), only the merging of them into the
        global dictionary is computed.
     */

php ArcTool.php show bundle_name start num
php ArcTool.php show double_index_name which_bundle start num
    /* outputs items start through num from bundle_name or name of Yioop or
       non-Yioop archive crawl folder */

php ArcTool.php user-impressions [key] [partition] [limit]
   /* displays impression log records from the ImpressionRecords log folder
      for a given user_id (key). If key is omitted, records for all users
      are returned. Optionally, you can specify which partition to read from
      and how many records to display. If no partition or limit is given,
      it will scan all partitions and return all matching records.
   */

php ArcTool.php item-impressions item_id [partition] [limit]
   /* displays impression log records from the ImpressionRecords log folder
      for a given item_id. Optionally, you can specify
      which partition to read from and how many records to display.
      If no partition or limit is given, it will use the latest partition and
      display all available records for that item_id.
    */

php ArcTool.php user-items user_id
    /* List all item IDs for the specified user. */

php ArcTool.php group-items group_id
    /* List all item IDs for the specified group. */

php ArcTool.php type-items item_type
   /*
      Lists all ITEM_IDs matching the specified type. The type may be
      given as a name (thread, wiki, group, query, cache, resource)
      or as its numeric code (1–6). Outputs one ID per line, or
      "No Items of that type" if none are found.
    */

EOD;
        exit();
    }
}
$arc_tool = new ArcTool();
$arc_tool->start();
