<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* A library of string, error reporting, log, hash, time, and conversion
* functions
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
use seekquarry\yioop\configs as C;
use seekquarry\atto as A;
/** For Yioop global defines */
require_once __DIR__ . "/../configs/Config.php";
/**
* Adds delimiters to a regex that may or may not have them
*
* @param string $expression a regex
* @return string rgex with delimiters if not there
*/
function addRegexDelimiters($expression)
{
$first = $expression;
$len = strlen($expression);
$last = $expression[$len - 1];
if (($first != $last && $len > 1) || $len == 1) {
$expression = ($first != '/' ) ?
"/" . $expression . "/"
: "@" . $expression . "@";
}
return $expression;
}
/**
* search for a pcre pattern in a subject from a given offset,
* return position of first match if found -1 otherwise.
*
* @param string $pattern a Perl compatible regular expression
* @param string $subject to search for pattern in
* @param int $offset character offset into $subject to begin searching from
* @param boolean $return_match whether to return as well what the
* match was for the pattern
* @return mixed if $return_match is false then the integer position of first
* match, otherwise, it returns the ordered pair [$pos, $match].
*/
function preg_search($pattern, $subject, $offset = 0, $return_match = false)
{
$pos = -1;
if (preg_match($pattern, $subject ?? "", $matches, PREG_OFFSET_CAPTURE,
$offset)) {
$pos = $matches[0][1];
}
if ($return_match) {
$match = (empty($matches[0][0])) ? [] : $matches[0][0];
return [$pos, $match];
}
return $pos;
}
/**
* Replaces a pcre pattern with a replacement in $subject starting from
* some offset.
*
* @param string $pattern a Perl compatible regular expression
* @param string $replacement what to replace the pattern with
* @param string $subject to search for pattern in
* @param int $offset character offset into $subject to begin searching from
* @return string result of the replacements
*/
function preg_offset_replace($pattern, $replacement, $subject, $offset = 0)
{
$subject ??= "";
$offset ??= 0;
$start = substr($subject, 0, $offset);
$end = substr($subject, $offset);
return $start . preg_replace($pattern, $replacement, $end);
}
/**
* Yioop replacement for parse_ini_file($name, true) in case
* parse_ini_file is on the disable_functions list. Name has underscores
* to match original function. This function
* checks if parse_ini_file is disabled on not. If not, it just
* calls parse_ini_file; otherwise, it simulates it enough so
* that configure.ini files used for string translations can be read.
*
* @param string $file filename of ini data to parse into an array
* @return array data parse from file
*/
function parse_ini_with_fallback($file)
{
static $disabled;
if (!file_exists($file)) {
return [];
}
if (!isset($disabled)) {
$disabled_string = ini_get('disable_functions');
$disabled = str_contains($disabled_string, "parse_ini_file");
}
if (!$disabled) {
return parse_ini_file($file, true);
}
$lines = file($file);
$name_space = null;
$ini = [];
$assigned = '"((?:[^"\\\]|\\\.)*)"|\w+|'."'".
'((?:[^'."'".'\\\]|\\\.)*)'."'";
foreach ($lines as $line) {
if (preg_match('/\[(\w+)\]/', $line, $matches)) {
$name_space = $matches[1];
$ini[$name_space] = [];
} else if (preg_match("/(\w+|(\w+)(\[\]))\s*\=\s*($assigned)/", $line,
$matches)){
if ($name_space) {
if ($matches[3] == '[]') {
$ini[$name_space][$matches[2]][] =
getIniAssignMatch($matches);
} else {
$ini[$name_space][$matches[1]] =
getIniAssignMatch($matches);
}
} else {
if ($matches[3] == '[]') {
$ini[$name_space][$matches[2]][] =
getIniAssignMatch($matches);
} else {
$ini[$matches[1]] =
getIniAssignMatch($matches);
}
}
}
}
return $ini;
}
/**
* Auxiliary function called from parse_ini_with_fallback to extract from
* the $matches array produced by the former function's preg_match
* what kind of assignment occurred in the ini file being parsed.
*
* @param string $matches produced by a preg_match in
* parse_ini_with_fallback
* @return mixed value of ini file assignment
*/
function getIniAssignMatch($matches)
{
if (isset($matches[6])) {
return $matches[6];
} else if (isset($matches[5])) {
return $matches[5];
} else if (isset($matches[4])) {
$tmp = $matches[4];
if ($tmp == "true") {
return true;
} else if ($tmp == "false") {
return false;
}
return $tmp;
}
return false;
}
/**
* Copies from $source string beginning at position $start, $length many
* bytes to destination string
*
* @param string $source string to copy from
* @param string &$destination string to copy to
* @param int $start starting offset
* @param int $length number of bytes to copy
* @param string $timeout_msg message to print if taking more
* than 30 seconds
*/
function charCopy($source, &$destination, $start, $length, $timeout_msg = "")
{
$endk = intval($length - 1);
$end = intval($start + $endk);
$start = intval($start);
if (strlen($destination) <= $endk) {
return;
}
if ($timeout_msg == "") {
for ($j = $end, $k = $endk; $j >= $start; $j--, $k--) {
$destination[$j] = $source[$k];
}
} else {
$time_out_check_frequency = 5000;
for ($j = $end, $k = $endk, $t = 0; $j >= $start; $j--, $k--, $t++) {
$destination[$j] = $source[$k];
if ($t > $time_out_check_frequency) {
crawlTimeoutLog($timeout_msg);
$t = 0;
}
}
}
}
/**
* Sleeps for up to $duration seconds, but in $interrupt_time second
* steps so a caller can wake early. After each step $wake_callback is
* consulted: when it returns true the sleep ends immediately,
* otherwise sleeping continues until $duration has elapsed. Lets a
* long idle back-off stay responsive to a stop request without busy
* waiting and without one large uninterruptible sleep.
*
* @param int $duration total seconds to sleep when never woken early
* @param int $interrupt_time seconds to sleep between each check;
* clamped to at least one second
* @param callable $wake_callback called after each step; returning
* true wakes the sleeper, false keeps it asleep. When omitted
* the call behaves like a plain sleep of $duration seconds
* @return int the number of seconds actually slept
*/
function interruptibleSleep($duration, $interrupt_time = 1,
$wake_callback = null)
{
$step = max(1, intval($interrupt_time));
$slept = 0;
while ($slept < $duration) {
$this_step = min($step, $duration - $slept);
sleep($this_step);
$slept += $this_step;
if ($wake_callback !== null && $wake_callback()) {
break;
}
}
return $slept;
}
/**
* Encodes an integer using variable byte coding.
*
* @param int $pos_int integer to encode
* @return string a string of 1-5 chars depending on how bit $pos_int was
*/
function vByteEncode($pos_int)
{
$result = ($pos_int < 128) ? chr($pos_int & 127) :
chr(128 | ($pos_int & 127));
$pos_int >>= 7;
while($pos_int > 0) {
$result .= ($pos_int < 128) ? chr($pos_int & 127) :
chr(128 | ($pos_int & 127));
$pos_int >>= 7;
}
return $result;
}
/**
* Decodes from a string using variable byte coding an integer.
*
* @param string $str string to use for decoding
* @param int &$offset byte offset into string when var int stored
* @return int the decoded integer
*/
function vByteDecode($str, &$offset)
{
$pos_int = ord($str[$offset]) & 127;
$shift = 7;
while ((ord($str[$offset++]) & 128) > 0) {
$pos_int += ((ord($str[$offset]) & 127) << $shift);
$shift += 7;
}
return $pos_int;
}
/**
* Appends a number re-encoded in unary to the end of an input string starting
* at a given bit offset into the string. Here n in unary has bit
* representation n-1 0's followed by a 1.
*
* @param int $number number to append
* @param string $input the string to append to
* @param int &$start_bit_offset by-reference bit offset into $input where
* the unary code should be written; updated to point past the new
* code on return
* @param bool $just_bit_offset whether to return the resulting string or
* just compute the number of bits long the resulting string would be
* @return mixed either the resulting string or its length
*/
function appendUnary($number, $input, &$start_bit_offset,
$just_bit_offset = false)
{
$total_bits = $start_bit_offset + $number;
if ($just_bit_offset) {
// only compute new bit offset
$start_bit_offset = $total_bits;
return $input;
}
$bits_last_byte = $total_bits & 7;
$total_chars = ($bits_last_byte == 0) ? ($total_bits >> 3) :
(($total_bits >> 3) + 1);
$bits_first_byte = $start_bit_offset & 7;
$start_char = $start_bit_offset >> 3;
$output = str_pad(substr($input, 0 , $start_char + 1), $total_chars,
"\x00");
$start_char_ord = ord($output[$start_char] ?? 0);
$start_char_ord &= ((1 << $bits_first_byte) - 1) << (8 - $bits_first_byte);
$output[$start_char] = chr($start_char_ord);
$last_ord = ord($output[$total_chars - 1]);
$output[$total_chars - 1] = chr($last_ord + (($bits_last_byte == 0) ? 1 :
(1 << (8 - $bits_last_byte))));
$start_bit_offset = $total_bits;
return $output;
}
/**
* Decodes a unary number from an input string at a given bit offset. Here n
* in unary has bit representation n-1 0's followed by a 1.
*
* @param string $input the string that we want to decode a unary number from
* @param int &$start_bit_offset the starting bit offset in $input to start
* decoding from. After the call it will be the position after the decode
* @return int the decoded unary number
*/
function decodeUnary($input, &$start_bit_offset)
{
$cur_char = $start_bit_offset >> 3;
if (!isset($input[$cur_char])) {
return false;
}
$bits_first_byte = $start_bit_offset & 7;
$cur_ord = (ord($input[$cur_char]) << $bits_first_byte) & 255;
if ($cur_ord > 0) {
$decoded_number = 9 - ceil(log($cur_ord + 1, 2));
$start_bit_offset += $decoded_number;
return $decoded_number;
}
$decoded_number = 8 - $bits_first_byte;
do {
$cur_char++;
$cur_ord = ord($input[$cur_char] ?? "\0");
$decoded_number += ($cur_ord == 0) ? 8 : 9 - ceil(log($cur_ord + 1, 2));
} while ($cur_ord == 0 && isset($input[$cur_char]));
$start_bit_offset += $decoded_number;
return $decoded_number;
}
/**
* Appends $num_bits bits from the start of the binary rep of $number
* beginning at offset $start_bit_offset of $input string overwriting any bits
* present. If $num_bits == -1, then appends all of $number.
*
* @param int $number to append
* @param string $input the string to append to.
* @param int &$start_bit_offset starting location to begin append from
* @param $num_bits number of bits of $input to append.
* @return string resulting string
*/
function appendBits($number, $input, &$start_bit_offset, $num_bits = -1)
{
$start_char = $start_bit_offset >> 3;
$num_bits = ($num_bits == -1) ? ceil(log($number + 1, 2)) : $num_bits;
$total_bits = $start_bit_offset + $num_bits;
$bits_last_byte = $total_bits & 7;
$total_chars = ($bits_last_byte == 0) ? ($total_bits >> 3) :
(($total_bits >> 3) + 1);
$number &= ((1 << $num_bits) - 1); // notice using low order here
$output = str_pad(substr($input, 0 , $start_char + 1), $total_chars,
"\x00");
$cur_char = $total_chars - 1;
$cur_bits = $number & ((1 << $bits_last_byte) - 1);
$number >>= $bits_last_byte;
$start_remaining_bits = $num_bits - $bits_last_byte;
$shift_last_byte = ($bits_last_byte == 0) ? 0 : 8 - $bits_last_byte;
$output[$cur_char] = chr(ord($output[$cur_char]) +
($cur_bits << $shift_last_byte));
$cur_char--;
for ($remaining_bits = $start_remaining_bits; $remaining_bits > 7 ;
$remaining_bits -= 8, $cur_char--, $number >>= 8) {
$output[$cur_char] = chr($number & 255);
}
if ($remaining_bits > 0) {
$start_char_ord = ord($output[$start_char]);
$start_char_ord &= (255 - (1 << ($remaining_bits - 1)));
$output[$start_char] = chr($start_char_ord + $number);
}
$start_bit_offset = $total_bits;
return $output;
}
/**
* Decode $num_bits many bits from the $input string beginning at offset
* $start_bit_offset. The result of this operation is up $start_bit_offset
* by number of bits that were able to be decoded.
*
* @param string $input string to decode bits from
* @param int &$start_bit_offset bit offset to start decoding from in $input
* @param int $num_bits number of bits tot try to decode
* @return int the number decoded
*/
function decodeBits($input, &$start_bit_offset, $num_bits)
{
$cur_char = $start_bit_offset >> 3;
$total_bits = $start_bit_offset + $num_bits;
$bits_first_byte = $start_bit_offset & 7;
$input_char = $input[$cur_char] ?? '';
$cur_ord = ((ord($input_char) << $bits_first_byte) & 255) >>
$bits_first_byte;
$output = $cur_ord & ((1 << (8 - $bits_first_byte)) - 1);
if ($num_bits <= 8 - $bits_first_byte) {
$excess_bits = 8 - $bits_first_byte - $num_bits;
$output >>= $excess_bits;
$start_bit_offset = $total_bits;
return $output;
}
$remaining_bits = $num_bits - (8 - $bits_first_byte);
$cur_char++;
while ($remaining_bits > 7 && isset($input[$cur_char])) {
$output <<= 8;
$output += ord($input[$cur_char]);
$remaining_bits -= 8;
$cur_char++;
}
if ($remaining_bits > 0 && isset($input[$cur_char])) {
$last_char_ord = ord($input[$cur_char]);
$output <<= $remaining_bits;
$output += ($last_char_ord >> (8 - $remaining_bits));
}
$start_bit_offset = $total_bits;
return $output;
}
/**
* Appends gamma code of $number beginning at offset $start_bit_offset of
* $input string overwriting any bits present. $start_bit_offset is updated
* to bit position after append.
*
* @param int $number to append
* @param string $input the string to append to.
* @param int &$start_bit_offset starting bit location to begin append from
* @return string resulting string
*/
function appendGamma($number, $input, &$start_bit_offset)
{
$bit_len = ceil(log($number + 1, 2));
appendUnary($bit_len, $input, $start_bit_offset, true);
$start_bit_offset--;
/*
$start_bit_offset will have advanced by appendUnary call
*/
return appendBits($number, $input, $start_bit_offset, $bit_len);
}
/**
* Decodes up to $num_decode gamma encoded integers beginning at
* $start_bit_offset. $start_bit_offset is updated to the bit position
* after the decoded integers.
*
* @param string $input the string to decode from
* @param int &$start_bit_offset starting bit location to decode from
* @param int $num_decode number of int's to decode
* @return array decoded int's
*/
function decodeGammaList($input, &$start_bit_offset, $num_decode)
{
$out_list = [];
for ($i = 0; $i < $num_decode; $i++) {
$num_bits = decodeUnary($input, $start_bit_offset);
$start_bit_offset--;
$out_list[] = decodeBits($input, $start_bit_offset, $num_bits);
}
return $out_list;
}
/**
* Appends using a Rice coding a sequence of integers $int_sequence at
* offset $start_bit_offset to the string $output, overwriting any bits present
* at that location. $start_bit_offset is updated to bit position after append.
* Encoding is done as a difference list. If $delta_start is set to
* a value other than >= then the first gap is assumed to be from int
* $delta_start
*
* @param array $int_sequence int's to append
* @param int $modulus i in the 2^i modulus to use for Rice code
* @param string $output the string to append to.
* @param int &$start_bit_offset starting bit location to begin append from
* @param int $delta_start if >= 0 previous int to use for difference list
* otherwise the first integer is encoded as itself rather than a difference
* @return string resulting string
*/
function appendRiceSequence($int_sequence, $modulus, $output,
&$start_bit_offset, $delta_start = -1)
{
$last_encode = $delta_start;
$output = appendUnary($modulus, $output, $start_bit_offset);
$mask = (1 << $modulus) - 1;
$i= 0;
foreach ($int_sequence as $pre_to_encode) {
$to_encode = ($delta_start < 0) ? $pre_to_encode :
$pre_to_encode - $last_encode;
$to_encode--;
$last_encode = $pre_to_encode;
$output = appendUnary(($to_encode >> $modulus) + 1, $output,
$start_bit_offset);
$output = appendBits($to_encode & $mask, $output, $start_bit_offset,
$modulus);
}
return $output;
}
/**
* Decodes up to $num_decode rice encoded difference list of integers beginning
* at $start_bit_offset. $start_bit_offset is updated to the bit position
* after the decoded integers. If $delta_start >= 0 then the first int is
* assumed to be the difference from $delta_start;
*
* @param string $input the string to decode from
* @param int &$start_bit_offset starting bit location to decode from
* @param int $num_decode number of int's to decode
* @param int $delta_start if >= 0 previous int to use for difference list
* otherwise the first integer is decoded as itself rather than a difference
* @return array decoded int's
*/
function decodeRiceSequence($input, &$start_bit_offset, $num_decode,
$delta_start = -1)
{
$out_list = [];
$last_decode = $delta_start;
$modulus = decodeUnary($input, $start_bit_offset);
for($i = 0; $i < $num_decode; $i++) {
$quotient = decodeUnary($input, $start_bit_offset);
/*
decodeUnary returns false once the bit offset has passed
the end of the input. Treating that as a quotient of zero
keeps the decoded value the same as before (a false
quotient already contributed zero to the shift below) while
avoiding a decrement applied to a boolean, which PHP warns
about and which a future PHP version will stop allowing.
This sits on the query hot path, so this branch and a few
other forms were benchmarked: against a full decode of a
real posting list the check costs nothing measurable, since
the surrounding unary and bit reads dominate the time.
*/
if ($quotient === false) {
$quotient = 0;
} else {
--$quotient;
}
$remainder = decodeBits($input, $start_bit_offset, $modulus);
$decode = ($quotient << $modulus) + $remainder + 1;
$last_decode = ($delta_start == -1) ? $decode : $last_decode + $decode;
$out_list[] = $last_decode;
}
return $out_list;
}
/**
* Encodes a list of integer positions of a term in a document. This
* is done as a gamma code of the first integer followed by the Rice
* coding of the remaining integers using a modulus based on the average
* gap between integers. If the number of positions is 1 or 2 then a gamma
* of each position only is used.
*
* @param array $positions integer term positions
* @return string encoded position list
*/
function encodePositionList($positions)
{
$output = "";
$num_positions = count($positions);
$start_bit_offset = 0;
$start_position = $positions[0];
$last = 0;
if ($num_positions < 3) {
foreach ($positions as $position) {
$output = appendGamma($position - $last, $output,
$start_bit_offset);
$last = $position;
}
return $output;
}
$average_gap = ($positions[$num_positions - 1] - $positions[0]) /
$num_positions;
$average_len = max(ceil(log($average_gap + 1, 2)), 2);
$output = appendGamma($start_position, $output, $start_bit_offset);
array_shift($positions);
return appendRiceSequence($positions, $average_len, $output,
$start_bit_offset, $start_position);
}
/**
* Decodes up to $num_decode term in document position integers from
* string $input under the assumption $input is encoded as per
* @see encodePositionList .
*
* @param string $input string to decode from
* @param int $num_decode number of integer to decode
* @return array decoded positions
*/
function decodePositionList($input, $num_decode)
{
$start_bit_offset = 0;
if ($num_decode < 3) {
$out_list = decodeGammaList($input, $start_bit_offset, $num_decode);
if (count($out_list) == 2) {
$out_list[1] += $out_list[0];
}
return $out_list;
}
$first = decodeGammaList($input, $start_bit_offset, 1)[0];
$out_list = decodeRiceSequence($input, $start_bit_offset, $num_decode - 1,
$first);
array_unshift($out_list, $first);
return $out_list;
}
/**
* Recodes a string in a 1-1 fashion to a string not involving
* \xFF (255). I.e., it maps characters \xFE -> \xFE\FD and \xFF -> \xFE\FE
*
* @param string $str to be encoded
* @return string encoded string without \xFF
*/
function encode255($str)
{
return strtr($str, ["\xFE" => "\xFE\xFD", "\xFF" => "\xFE\xFE"]);
}
/**
* Decodes a string in a 1-1 fashion from a string not involving
* \xFF (255). I.e., it maps characters \xFE\FE -> \xFF and \xFE\FD -> \xFF
*
* @param string $str to be frcoded
* @return string decoded string
*/
function decode255($str)
{
return strtr($str, ["\xFE\xFE"=>"\xFF", "\xFE\xFD" => "\xFE"]);
}
/**
* Recodes a string in a 1-1 fashion to a string not involving underscore
* (_). I.e., it maps characters - -> -- and _ -> -=
*
* @param string $str to be encoded
* @return string encoded string without _
*/
function encodeUnderscore($str)
{
return strtr(($str ?? ""), ["-" => "--", "_" => "-="]);
}
/**
* Decodes a string in a 1-1 fashion from a string not involving underscore
* (_). I.e., it maps characters -= -> _ and -- -> -
*
* @param string $str to be frcoded
* @return string decoded string
*/
function decodeUnderscore($str)
{
return strtr($str, ["-="=>"_", "--" => "-"]);
}
/**
* Encodes a list of strings as their @see encode255 versions separated by
* \xFF's
*
* @param array $strs strings to encode as a single string
* @return string encoded list
*/
function packEncode255($strs)
{
return implode("\xFF", array_map($strs, encode255));
}
/**
* Decodes a list of strings from a string that encoded as their @see encode255
* of its elements separated by \xFF's
*
* @param string $encoded_strs string to decode into a list of strings
* @return array decoded list
*/
function unpackDecode255($encoded_strs)
{
return array_map(explode("\xFF", $encoded_strs), decode255);
}
/**
* Makes an packed integer string from a docindex and the number of
* occurrences of a word in the document with that docindex.
*
* @param int $doc_index index (i.e., a count of which document it
* is rather than a byte offset) of a document in the document string
* @param array $position_list integer positions word occurred in that doc
* @param bool $delta if true then stores the position_list as a sequence of
* differences (a delta list)
* @return string a modified9 (our compression scheme) packed
* string containing this info.
*/
function packPosting($doc_index, $position_list, $delta = true)
{
if ($delta) {
$delta_list = deltaList($position_list);
} else {
$delta_list = $position_list;
}
if ( $doc_index >= (2 << 14) && isset($delta_list[0])
&& $delta_list[0] < (2 << 9) && $doc_index < (2 << 17)) {
$delta_list[0] += (((2 << 17) + $doc_index) << 9);
} else {
// we add 1 to doc_index to make sure not 0 (modified9 needs > 0)
array_unshift($delta_list, ($doc_index + 1));
}
$encoded_list = encodeModified9($delta_list);
return $encoded_list;
}
/**
* Given a packed integer string, uses the top three bytes to calculate
* a doc_index of a document in the shard, and uses the low order byte
* to computer a number of occurrences of a word in that document.
*
* @param string $posting a string containing
* a doc index position list pair coded encoded using modified9
* @param int &$offset a offset into the string where the modified9 posting
* is encoded
* @param bool $dedelta if true then assumes the list is a sequence of
* differences (a delta list) and undoes the difference to get
* the original sequence
* @return array consisting of integer doc_index and a subarray consisting
* of integer positions of word in doc.
*/
function unpackPosting($posting, &$offset, $dedelta = true)
{
$delta_list = (array) decodeModified9($posting, $offset);
$doc_index = array_shift($delta_list);
if (($doc_index & (2 << 26)) > 0) {
$delta0 = $doc_index;
$doc_index >>= 9;
$doc_index -= (2 << 17);
$delta0 -= (((2 << 17) + $doc_index) << 9);
array_unshift($delta_list, $delta0);
} else {
$doc_index--;
}
if ($dedelta) {
deDeltaList($delta_list);
}
return [$doc_index, $delta_list];
}
/**
* This method is used while appending one index shard to another.
* Given a string of postings adds $add_offset add to each offset to the
* document map in each posting.
*
* @param string &$postings a string of index shard postings
* @param int $add_offset an fixed amount to add to each postings doc map offset
*
* @return string $new_postings where each doc offset has had $add_offset added
* to it
*/
function addDocIndexPostings(&$postings, $add_offset)
{
$offset = 0;
$new_postings = "";
$postings_len = strlen($postings);
while($offset < $postings_len) {
$post_string = nextPostString($postings, $offset);
if ($post_string == "" || !($tmp = unpack("N*", $post_string))) {
continue;
}
$posting_list = call_user_func_array("array_merge",
array_map(C\NS_LIB . "unpackListModified9", $tmp));
if (!is_array($posting_list)) {
continue;
}
$doc_index = array_shift($posting_list);
if (($doc_index & (2 << 26)) > 0) {
$post0 = ($doc_index & ((2 << 9) - 1));
array_unshift($posting_list, $post0);
$doc_index -= $post0;
$doc_index -= (2 << 26);
$doc_index >>= 9;
} else {
$doc_index--;
}
$doc_index += $add_offset;
if ($doc_index >= (2 << 14) && isset($posting_list[0])
&& $posting_list[0] < (2 << 9) && $doc_index < (2 << 17)) {
$posting_list[0] += (((2 << 17) + $doc_index) << 9);
} else {
// we add 1 to doc_index to make sure not 0 (modified9 needs > 0)
array_unshift($posting_list, ($doc_index + 1));
}
$new_postings .= encodeModified9($posting_list);
}
return $new_postings;
}
/**
* Computes the difference of a list of integers.
* i.e., (a1, a2, a3, a4) becomes (a1, a2-a1, a3-a2, a4-a3)
*
* @param array $list a nondecreasing list of integers
* @return array the corresponding list of differences of adjacent
* integers
*/
function deltaList($list)
{
$last = 0;
$delta_list = [];
foreach ($list as $elt) {
$delta_list[] = $elt - $last;
$last = $elt;
}
return $delta_list;
}
/**
* Given an array of differences of integers reconstructs the
* original list. This computes the inverse of the deltaList function
*
* @see deltaList
* @param array $delta_list a list of nonegative integers
* @return array a nondecreasing list of integers
*/
function deDeltaList(&$delta_list)
{
$last = 0;
$num = count($delta_list);
for ($i = 1; $i < $num; $i++) {
$delta_list[$i] += $delta_list[$i - 1];
}
}
/**
* Mini-class (so not own file) used to hold encode decode info related to
* Mod9 encoding (as variant of Simplified-9 specify to Yioop).
* Mod9 is used to incode a sequence of positive (greater than 0) integers
* as a string. WARNING: do not expect is to work/decode correctly if
* sequence has a 0 as the decoding process assumes 0 indicates end of sequence.
* @see encodeModified9 for a complete description
*/
class Mod9Constants
{
/**
* Used in Modified 9 encoding. The ith array entry represents the number of
* i bit elements that can be stored in a word using modified 9 (0th index
* location is a dummy value 0 as can't store 0 bit numbers)
* @var array
*/
public static $MOD9_PACK_POSSIBILITIES = [
0, 24, 12, 7, 6, 5, 4, 3, 3, 3, 2, 2, 2, 2,
2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1];
/**
* Used in Modified 9 encoding. Key values are the number of elements we
* would like to store in the current word. Values are the bit prefix to use
* on first byte of word. Notices bits 7 and 6 (128 and 64) are not parts of
* prefixes as used for continuation bits.
* @var array
*/
public static $MOD9_NUM_ELTS_CODES = [
24 => 63, 12 => 62, 7 => 60, 6 => 56, 5 => 52, 4 => 48, 3 => 32,
2 => 16, 1 => 0];
/**
* Keys of this array are prefix codes from the high order byte of a word
* encoded using Modified 9, values are the number of bits used to encode
* an element if that prefix code was used.
* @var array
*/
public static $MOD9_NUM_BITS_CODES = [63 => 1, 62 => 2, 60 => 3, 56 => 4,
52 => 5, 48 => 6, 32 => 9, 16 => 14, 0 => 28];
/**
* Keys of this array are prefix codes from the high order byte of a word
* encoded using Modified 9, values are the number of elts stored in the
* remaining bits of the word
* @var array
*/
public static $MOD9_NUM_ELTS_DECODES = [
63 => 24, 62 => 12, 60=> 7, 56 => 6, 52 => 5, 48 => 4, 32 => 3,
16 => 2, 0 => 1];
}
/**
* Encodes a sequence of integers x, such that 1 <= x <= 2<<28-1
* as a string. NOTICE x>=1.
*
* The encoded string is a sequence of 4 byte words (packed int's).
* The high order 2 bits of a given word indicate whether or not
* to look at the next word. The codes are as follows:
* 11 start of encoded string, 10 continue four more bytes, 01 end of
* encoded, and 00 indicates whole sequence encoded in one word.
*
* After the high order 2 bits, the next most significant bits indicate
* the format of the current word. There are nine possibilities:
* 00 - 1 28 bit number, 01 - 2 14 bit numbers, 10 - 3 9 bit numbers,
* 1100 - 4 6 bit numbers, 1101 - 5 5 bit numbers, 1110 6 4 bit numbers,
* 11110 - 7 3 bit numbers, 111110 - 12 2 bit numbers, 111111 - 24 1 bit
* numbers.
*
* @param array $list a list of positive integers satsfying above
* @return string encoded string
*/
function encodeModified9($list)
{
$MOD9_PACK_POSSIBILITIES = Mod9Constants::$MOD9_PACK_POSSIBILITIES;
$cnt = 0;
$cur_size = 1;
$cur_len = 1;
$pack_list = [];
$list_string = "";
$continue_bits = 3;
foreach ($list as $elt) {
$old_len = $cur_len;
while($elt > $cur_size)
{
$cur_len++;
$cur_size = (1 << $cur_len) - 1;
}
if ($cnt < $MOD9_PACK_POSSIBILITIES[$cur_len]) {
$pack_list[] = $elt;
$cnt++;
} else {
$list_string .= packListModified9($continue_bits,
$MOD9_PACK_POSSIBILITIES[$old_len], $pack_list);
$continue_bits = 2;
$pack_list = [$elt];
$cur_size = 1;
$cur_len = 1;
$cnt = 1;
while($elt > $cur_size)
{
$cur_size = (1 << $cur_len) - 1;
$cur_len++;
}
}
}
$continue_bits = ($continue_bits == 3) ? 0 : 1;
$list_string .= packListModified9($continue_bits,
$MOD9_PACK_POSSIBILITIES[$cur_len], $pack_list);
return $list_string;
}
/**
* Packs the contents of a single word of a sequence being encoded
* using Modified9.
*
* @param int $continue_bits the high order 2 bits of the word
* @param int $cnt the number of element that will be packed in this word
* @param array $pack_list a list of positive integers to pack into word
* @return string encoded 4 byte string
* @see encodeModified9
*/
function packListModified9($continue_bits, $cnt, $pack_list)
{
$MOD9_NUM_ELTS_CODES = Mod9Constants::$MOD9_NUM_ELTS_CODES;
$MOD9_NUM_BITS_CODES = Mod9Constants::$MOD9_NUM_BITS_CODES;
$out_int = 0;
$code = $MOD9_NUM_ELTS_CODES[$cnt];
$num_bits = $MOD9_NUM_BITS_CODES[$code];
foreach ($pack_list as $elt) {
$out_int <<= $num_bits;
$out_int += $elt;
}
$out_string = packInt($out_int);
$out_string[0] = chr(($continue_bits << 6) + $code + ord($out_string[0]));
return $out_string;
}
/**
* Returns the next complete posting string from $input_string being at offset.
* Does not do any decoding.
*
* @param string &$input_string a string of postings
* @param int &$offset an offset to this string which will be updated after call
* @return string undecoded posting
*/
function nextPostString(&$input_string, &$offset)
{
if (!isset($input_string[$offset + 3])) {
$offset +=4; //make sure offset always increases to be safe
return "";
}
$flag_mask = 192;
$continue_threshold = 128;
$len = strlen($input_string);
$end = $offset;
$error = false;
$flag_bits = (ord($input_string[$end]) & $flag_mask) ;
if ($flag_bits && $flag_bits != $flag_mask) {
crawlLog("!! Decode Error Flags: $flag_bits $flag_mask");
crawlLog("!! Dropping posting at $offset cycle to next posting.");
crawlLog("!! Dropped posting length $len.");
$offset += 4;
return "";
}
$end += 4;
while($end < $len && $flag_bits >= $continue_threshold) {
$flag_bits = (ord($input_string[$end]) & $flag_mask);
$end += 4;
}
$post_string = substr($input_string, $offset, $end - $offset);
$offset = $end;
return $post_string;
}
/**
* Decoded a sequence of positive integers from a string that has been
* encoded using Modified 9
*
* @param string $input_string string to decode from
* @param int &$offset where to string in the string, after decode
* points to where one was after decoding.
* @return array sequence of positive integers that were decoded
* @see encodeModified9
*/
function decodeModified9($input_string, &$offset)
{
$post_string = nextPostString($input_string, $offset);
return call_user_func_array( "array_merge",
array_map(C\NS_LIB . "unpackListModified9",
unpack("N*", $post_string)));
}
if (!extension_loaded("yioop") ) {
/**
* Decode a single word with high two bits off according to modified 9
*
* @param string $encoded_list four byte string to decode
* @return array sequence of integers that results from the decoding.
*/
function unpackListModified9($encoded_list)
{
switch ($encoded_list & 0x30000000) {
case 0:
return [$encoded_list & 0x0FFFFFFF]; //lop off high nibble
break;
case 0x10000000:
$encoded_list &= 0xEFFFFFFF;
$num_bits = 14;
$num_elts = 2;
$mask = 0x3FFF;
$shift = 14;
break;
case 0x20000000:
$encoded_list &= 0xDFFFFFFF;
$num_bits = 9;
$num_elts = 3;
$mask = 0x1FF;
$shift = 18;
break;
default:
$MOD9_NUM_BITS_CODES = Mod9Constants::$MOD9_NUM_BITS_CODES;
$MOD9_NUM_ELTS_DECODES = Mod9Constants::$MOD9_NUM_ELTS_DECODES;
$int_string = packInt($encoded_list);
$first_char = ord($int_string[0]);
foreach ($MOD9_NUM_BITS_CODES as $code => $num_bits) {
if (($first_char & $code) == $code) {
break;
}
}
$num_elts = $MOD9_NUM_ELTS_DECODES[$code];
$mask = (1 << $num_bits) - 1;
$int_string[0] = chr($first_char - $code);
$encoded_list = unpackInt($int_string);
}
$decoded_list = [];
for ($i = 0; $i < $num_elts; $i++) {
if (($pre_elt = $encoded_list & $mask) == 0) {
break;
}
array_unshift($decoded_list, $pre_elt);
$encoded_list >>= $num_bits;
}
return $decoded_list;
}
/**
* Given an int encoding encoding a doc_index followed by a position
* list using Modified 9, extracts just the doc_index.
*
* @param int $encoded_list in the just described format
* @return int a doc index into an index shard document map.
*/
function docIndexModified9($encoded_list)
{
$t26 = 2 << 26;
switch ($encoded_list & 0x30000000) {
case 0:
$encoded_list &= 0x0FFFFFFF; //lop off high nibble
return (($encoded_list & $t26) > 0) ?
($encoded_list - $t26 + ($encoded_list & 0x1FF)) >> 9 :
$encoded_list - 1;
break;
case 0x10000000:
$encoded_list &= 0xEFFFFFFF;
$num_bits = 14;
$mask = 0x3FFF;
$shift = 14;
break;
case 0x20000000:
$encoded_list &= 0xDFFFFFFF;
$num_bits = 9;
$mask = 0x1FF;
$shift = 18;
break;
default:
$MOD9_NUM_BITS_CODES = Mod9Constants::$MOD9_NUM_BITS_CODES;
$MOD9_NUM_ELTS_DECODES = Mod9Constants::$MOD9_NUM_ELTS_DECODES;
$first_char = $encoded_list >> 24;
foreach ($MOD9_NUM_BITS_CODES as $code => $num_bits) {
if (($first_char & $code) == $code) break;
}
$num_elts = $MOD9_NUM_ELTS_DECODES[$code];
$mask = (1 << $num_bits) - 1;
$shift = $num_bits * ($num_elts - 1);
$int_string = packInt($encoded_list);
$int_string[0] = chr($first_char - $code);
$encoded_list = (int)hexdec(bin2hex($int_string));
}
do {
if ($doc_index = (($encoded_list >> $shift) & $mask)) {
$doc_index -= (($doc_index & $t26) > 0) ?
$t26 + ($doc_index & 0x1FF) : 1;
return $doc_index;
}
$shift -= $num_bits;
} while($shift >= 0);
return $doc_index; //shouldn't get here
}
/**
* Unpacks an int from a 4 char string
*
* @param string $str where to extract int from
* @return int extracted integer
*/
function unpackInt($str)
{
if (is_string($str)) {
return (int)hexdec(bin2hex($str));
}
return false;
}
}// end extension_loaded check
/**
* Packs an int into a 4 char string
*
* @param int $my_int the integer to pack
* @return string the packed string
*/
function packInt($my_int)
{
return pack("N", $my_int);
}
/**
* Unpacks a float from a 4 char string
*
* @param string $str where to extract int from
* @return float extracted float
*/
function unpackFloat($str)
{
if (!is_string($str)) return false;
$tmp = unpack("G", $str);
return $tmp[1];
}
/**
* Packs an float into a four char string
*
* @param float $my_float the float to pack
* @return string the packed string
*/
function packFloat($my_float)
{
return pack("G", $my_float);
}
/**
* Used to change the namespace of a serialized php object (assumes doesn't
* have nested subobjects)
*
* @param string $class_name new fully qualified name with namespace
* @param string $object_string serialized object
*
* @return string serialized object with new name
*/
function renameSerializedObject($class_name, $object_string)
{
/* number of digits in the length of name of the object needs to be
less than 12 digits (probably more like 4) for this to work.
*/
$name_length = intval(substr($object_string, 2, 14));
$name_space_info_length = strlen("O:".$name_length.":") +
$name_length + 2; // 2 for quotes;
$object_string = 'O:' .
strlen($class_name) . ':"'. $class_name.'"' .
substr($object_string, $name_space_info_length);
return $object_string;
}
/**
* Parses a provided string to make a DOM object. First tries to parse
* using XML and if this fails uses the more robust HTML Dom parser
* and manipulates the resulting DOM tree to make correspond to original
* tags for XML that isn't HTML
*
* @param string $to_parse the string to parse a DOMDocument from
* @return DOMDocument computed based on the provided string
*/
function getDomFromString($to_parse)
{
set_error_handler(null);
$dom = new \DOMDocument("1.0");
$dom->formatOutput = true;
$dom->preserveWhiteSpace = false;
if (stristr($to_parse, "<html") === false && !empty($to_parse)) {
@$dom->loadXML($to_parse);
}
if ($dom->documentElement == null) {
$dom = new \DOMDocument("1.0");
$dom->formatOutput = true;
$dom->preserveWhiteSpace = false;
//this hack modified from php.net
@$dom->loadHTML('<?xml encoding="UTF-8">' . $to_parse);
foreach ($dom->childNodes as $item) {
if ($item->nodeType == XML_PI_NODE) {
$dom->removeChild($item); // remove hack
}
}
$dom->encoding = 'UTF-8';
if (!empty($dom->documentElement->firstChild) &&
!empty($dom->documentElement->firstChild->tagName) &&
$dom->documentElement->firstChild->tagName == 'body' &&
!empty($dom->documentElement->firstChild->firstChild)) {
$node = $dom->documentElement->firstChild->firstChild;
$to_parse = $dom->saveXML($node);
if (!empty($to_parse)) {
$dom = new \DOMDocument("1.0");
$dom->formatOutput = true;
$dom->preserveWhiteSpace = false;
@$dom->loadXML($to_parse);
if ($dom->documentElement == null) {
$dom = new \DOMDocument("1.0");
$dom->formatOutput = true;
@$dom->loadHTML($to_parse);
}
}
}
}
restore_error_handler();
return $dom;
}
/**
* Returns an array of DOMDocuments for the nodes that match an xpath
* query on $dom, a DOMDocument
*
* @param DOMDocument $dom document to run xpath query on
* @param string $query xpath query to run
* @return array of DOMDocuments one for each node matching the
* xpath query in the original DOMDocument
*/
function getTags($dom, $query)
{
$nodes = [];
if (is_a($dom, "DOMDocument")) {
$dom_xpath = new \DOMXPath($dom);
if (!$dom_xpath) {
return [];
}
$tags = $dom_xpath->query($query);
}
if (empty($tags) || $tags->length == 0) {
if(!empty($dom->textContent)) {
$redom = new \DOMDocument;
@$redom->loadHTML($dom->textContent);
$dom_xpath = new \DOMXPath($redom);
if (!$dom_xpath) {
return [];
}
$tags = $dom_xpath->query($query);
}
if (empty($tags) || $tags->length == 0) {
return [];
}
}
$i = 0;
while ($item = $tags->item($i)) {
$tmp_dom = new \DOMDocument;
$tmp_dom->formatOutput = true;
$node = $tmp_dom->importNode($item, true);
$tmp_dom->appendChild($node);
$nodes[] = $tmp_dom;
$i++;
}
return $nodes;
}
/**
* Converts a string to string where each char has been replaced by its
* hexadecimal equivalent
*
* @param string $str what we want rewritten in hex
* @return string the hexified string
*/
function toHexString($str)
{
$str ??= "";
if (!is_string($str)) {
trigger_error("toHexString called with non-string:" . serialize($str));
return "";
}
$out = "";
for ($i = 0; $i < strlen($str); $i++) {
$out .= sprintf("%02X",ord($str[$i]))." ";
}
return $out;
}
/**
* Converts a string to string where each char has been replaced by a Integer
* equivalent
*
* @param string $str what we want rewritten in hex
* @return string the hexified string
*/
function toIntString($str)
{
if (!is_string($str)) {
trigger_error("toIntString called with non-string:" . serialize($str));
return "";
}
$out = "";
for ($i = 0; $i < strlen($str); $i++) {
$out .= sprintf("%03u",ord($str[$i]))." ";
}
return $out;
}
/**
* Converts a string to string where each char has been replaced by its
* binary equivalent
*
* @param string $str what we want rewritten in hex
* @return string the binary string
*/
function toBinString($str)
{
if (!is_string($str)) {
trigger_error("toBinString called with non-string:" . serialize($str));
return "";
}
$out = "";
for ($i = 0; $i < strlen($str); $i++) {
$out .= substr(decbin(256+ord($str[$i])), 1)." ";
}
return $out;
}
/**
* Subtract two unsigned binary strings: $a - $b
*
* @param string $a Minuend, big-endian binary string
* @param string $b Subtrahend, big-endian binary string
* @return array [int $sign, string $magnitude]
* $sign = 1 if non-negative, -1 if negative
* $magnitude = absolute difference as binary string
*/
function bigSubtract(string $a, string $b): array
{
$a = ltrim($a, "\x00") ?: "\x00";
$b = ltrim($b, "\x00") ?: "\x00";
if ($a === $b) {
return [1, "\x00"];
}
$wa = unpackWords($a);
$wb = unpackWords($b);
// Determine sign
if (compareWords($wa, $wb) < 0) {
[$wa, $wb] = [$wb, $wa];
$sign = -1;
} else {
$sign = 1;
}
$na = count($wa);
$nb = count($wb);
$wr = [];
$borrow = 0;
for ($i = 0; $i < $na; $i++) {
$ai = $wa[$i];
$bi = $i < $nb ? $wb[$i] : 0;
$val = $ai - $bi - $borrow;
if ($val < 0) {
$val += 0x100000000; // 2^32
$borrow = 1;
} else {
$borrow = 0;
}
$wr[$i] = $val;
}
return [$sign, packWords($wr)];
}
/**
* Multiply two unsigned binary strings.
*
* @param string $a Multiplicand, big-endian binary string
* @param string $b Multiplier, big-endian binary string
* @return string Product as big-endian binary string
*/
function bigMultiply(string $a, string $b): string
{
$a = ltrim($a, "\x00");
$b = ltrim($b, "\x00");
if ($a === '' || $b === '') {
return "\x00";
}
$wa = unpackWords($a);
$wb = unpackWords($b);
$na = count($wa);
$nb = count($wb);
$wr = array_fill(0, $na + $nb, 0);
for ($i = 0; $i < $na; $i++) {
$carry = 0;
for ($j = 0; $j < $nb; $j++) {
[$lo, $hi] = mul32x32Safe($wa[$i], $wb[$j]);
$sum = $wr[$i+$j] + $lo + $carry;
$wr[$i+$j] = $sum & 0xFFFFFFFF; // safe
$carry = $hi + ($sum >> 32);
}
$wr[$i+$nb] += $carry;
}
return packWords($wr);
}
/**
* Multiply two 32-bit integers safely using 16-bit sub-limbs.
*
* @param int $a 32-bit integer
* @param int $b 32-bit integer
* @return int[] [lo, hi] 64-bit product as two 32-bit integers
*/
function mul32x32Safe(int $a, int $b): array
{
$limb_mask = 0xFFFF; // 16-bit mask
$limb_bits = 16;
$a_lo = $a & $limb_mask;
$a_hi = $a >> $limb_bits;
$b_lo = $b & $limb_mask;
$b_hi = $b >> $limb_bits;
$p0 = $a_lo * $b_lo;
$p1 = $a_lo * $b_hi;
$p2 = $a_hi * $b_lo;
$p3 = $a_hi * $b_hi;
$mid = $p1 + $p2 + ($p0 >> $limb_bits);
$lo = (($mid & $limb_mask) << $limb_bits) | ($p0 & $limb_mask);
$hi = $p3 + ($mid >> $limb_bits);
return [$lo, $hi];
}
/**
* Convert big-endian binary string to an array of 32-bit words.
* Words are returned in **little-endian internal order**.
*
* @param string $bin Binary string
* @return int[] Array of 32-bit words
*/
function unpackWords(string $bin): array
{
$pad = (4 - (strlen($bin) % 4)) % 4;
if ($pad) {
$bin = str_repeat("\x00", $pad) . $bin;
}
// Big-endian 32-bit words, reversed to little-endian
return array_reverse(array_values(unpack('N*', $bin)));
}
/**
* Convert little-endian 32-bit word array to a big-endian binary string.
*
* @param int[] $words Array of 32-bit words
* @return string Binary string
*/
function packWords(array $words): string
{
while (count($words) > 1 && end($words) === 0) {
array_pop($words);
}
$bin = pack('N*', ...array_reverse($words));
return ltrim($bin, "\x00") ?: "\x00";
}
/**
* Compare two little-endian 32-bit word arrays.
*
* @param int[] $a First word array
* @param int[] $b Second word array
* @return int -1 if $a < $b, 0 if equal, 1 if $a > $b
*/
function compareWords(array $a, array $b): int
{
$la = count($a);
$lb = count($b);
if ($la !== $lb) {
return $la < $lb ? -1 : 1;
}
for ($i = $la - 1; $i >= 0; $i--) {
if ($a[$i] !== $b[$i]) {
return $a[$i] < $b[$i] ? -1 : 1;
}
}
return 0;
}
/**
* Converts a string of the form some int followed by K, M, or G.
* into its integer equivalent. For example 4K would become 4000,
* 16M would become 16000000, and 1G would become 1000000000
* Note not using base 2 for K, M, G
*
* @param string $metric_num metric number to convert
* @return int number the metric string corresponded to
*/
function metricToInt($metric_num)
{
$num = intval($metric_num);
if (is_string($metric_num)) {
$last_char = $metric_num[strlen($metric_num) - 1];
switch (strtoupper($last_char)) {
case "K":
$num *= 1000;
break;
case "M":
$num *= pow(1000, 2);
break;
case "G":
$num *= pow(1000, 3);
break;
case "T":
$num *= pow(1000, 4);
break;
}
}
return $num;
}
/**
* The amount of memory, in bytes, that memory-sensitive code should
* treat as its working ceiling: the PHP memory_limit scaled down by a
* fill factor to leave headroom before the real limit. When the
* memory_limit is unlimited (the ini value of -1, common for the
* command line), there is no ceiling at all, so this returns the
* largest representable integer and callers never wrongly decide that
* memory is running low.
*
* @param float $fill_factor fraction of the limit to treat as the
* ceiling, for example MEMORY_FILL_FACTOR
* @return int|float the byte figure to compare memory_get_usage()
* against
*/
function memoryCeiling($fill_factor)
{
$limit = metricToInt(ini_get("memory_limit"));
if ($limit <= 0) {
return PHP_INT_MAX;
}
return $limit * $fill_factor;
}
/**
* Converts a number to a string followed by nothing, K, M, G, T
* depending on whether number is < 1000, < 10^6, < 10^9, or < 10^(12)
*
* @param int $num number to convert
* @return string number the metric string corresponded to
*/
function intToMetric($num)
{
if (is_int($num)) {
$metric_letters = ["", "K", "M", "G", "T", "P", "E"];
$power = ($num > 0) ? max(intval(log($num, 1000)), 0) : 0;
$power = (isset($metric_letters[$power])) ? $power : 6;
$num = round($num / pow(1000, $power)). $metric_letters[$power];
}
return $num;
}
/**
* Logs a message to a logfile or the screen. The super-global field
* $_SERVER['LOG_TO_FILES'] determines if this will log to a file. If not,
* then in cli mode, will log to stdout, otherwise it will use error_log. When
* logging to file $_SERVER["NO_ROTATE_LOGS"] controls whether or not there
* will be a log file rotation. The first call to this method is typically used
* to set up a process to check for liveness. For example a call:
* crawlLog("\n\nInitialize logger..", $this->process_name, true);
* says $this->process_name should be checked for liveness as part of any
* subsequent logging activity such as a call crawlLog("Another Message");
* (note subsequent call don't need to specify the process name).
*
* @param string $msg message to log. If empty then no message written
* @param string $lname name of log file in the LOG_DIR directory, rotated logs
* will also use this as their basename followed by a number followed by
* gzipped (since they are gzipped (older versions of Yioop used bzip
* Some distros don't have bzip but do have gzip. Also gzip was
* being used elsewhere in Yioop, so to remove the dependency bzip was
* replaced )).
* @param bool $check_process_handler by default set to false. After the first
* time set to true, as long as in subsequent calls set to false,
* processHandler will be called to check how long the code has run
* since the last time processHandler called.
*/
function crawlLog($msg, $lname = null, $check_process_handler = false)
{
static $logname;
static $last_check_time = 0;
static $check_handler = false;
if (!empty($_SERVER["NO_LOGGING"])) {
return;
}
if ($lname != null)
{
$logname = $lname;
} else if (!isset($logname)) {
$logname = "message";
}
if ($check_process_handler) {
$check_handler = $check_process_handler;
}
$out_msg = makeTimestamp() . " $msg";
if (!empty($_SERVER["LOG_TO_FILES"])) {
$pre_logfile = C\LOG_DIR . "/$logname";
$logfile = "$pre_logfile.log";
clearstatcache(); //hopefully, this doesn't slow things too much
if (empty($_SERVER["NO_ROTATE_LOGS"]) &&
file_exists($logfile) && filesize($logfile) > C\MAX_LOG_FILE_SIZE) {
$last_logfile = "$pre_logfile." . C\NUMBER_OF_LOG_FILES . ".log.gz";
if (file_exists($last_logfile)) {
unlink($last_logfile);
}
for ($i = C\NUMBER_OF_LOG_FILES; $i > 0; $i--) {
$previous_logfile = "$pre_logfile.".($i-1).".log.gz";
if (file_exists($previous_logfile)) {
rename($previous_logfile, "$pre_logfile.$i.log.gz");
}
}
file_put_contents("$pre_logfile.0.log.gz",
gzencode(file_get_contents($logfile)));
file_put_contents($logfile, "");
}
//don't use error_log options in this case to happify hiphop4php
file_put_contents($logfile, $out_msg . "\n", FILE_APPEND);
if (changeInMicrotime($last_check_time) > 5 &&
$check_handler && !$check_process_handler) {
$continue = ($logname == 'index') ? true : false;
CrawlDaemon::processHandler($continue);
$last_check_time = microtime(true);
}
} else if (php_sapi_name() != 'cli' &&
(!C\nsdefined("IS_OWN_WEB_SERVER") || (!empty($_SERVER['LOG_TO_FILES'])
&& $_SERVER['LOG_TO_FILES'] == "terminal"))) {
error_log($out_msg."\n");
} else {
file_put_contents("php://stdout", $out_msg . "\n", FILE_APPEND);
}
return;
}
/**
* Used to make a log file entry time string of format: entry number, time in r
* format.
* @param int $time a unix timestamp
* @return string [line_count_in_log r_formatted_date]
*/
function makeTimestamp($time = -1)
{
static $count = 0;
$count = ($count + 1) % 1000000000;
$time = ($time == -1) ? time() : $time;
$time_string = date("r", $time);
return "[$count $time_string]";
}
/**
* Writes a log message $msg if more than LOG_TIMEOUT time has passed since
* the last time crawlTimeoutLog was called. Useful in loops to write a message
* as progress is made through the loop (but not on every iteration, but
* say every 30 seconds).
*
* @param mixed $msg usually a string with what to be printed out after the
* timeout period. If $msg === true then clears the timeout cache
* @return bool whether a log message was written
*/
function crawlTimeoutLog($msg)
{
static $cache_time = 0;
if (changeInMicrotime($cache_time) < C\LOG_TIMEOUT) {
return false;
}
if (func_num_args() > 1) {
$out_msg = call_user_func_array('sprintf', func_get_args());
} else {
$out_msg = & $msg;
}
crawlLog($out_msg . " Current memory usage:".memory_get_usage());
$cache_time = microtime(true);
return true;
}
/**
* Computes an 8 byte hash of a string for use in storing documents.
*
* An eight byte hash was chosen so that the odds of collision even for
* a few billion documents via the birthday problem are still reasonable.
* If the raw flag is set to false then an 11 byte base64 encoding of the
* 8 byte hash is returned. The hash is calculated as the xor of the
* two halves of the 16 byte md5 of the string. (8 bytes takes less storage
* which is useful for keeping more doc info in memory)
*
* @param string $string the string to hash
* @param bool $raw whether to leave raw or base 64 encode
* @return string the hash of $string
*/
function crawlHash($string, $raw = false)
{
$pre_hash = md5($string, true);
$left = substr($pre_hash, 0, 8);
$right = substr($pre_hash, 8, 8);
$combine = $right ^ $left;
if (!$raw) {
$hash = base64Hash($combine);
// common variant of base64 safe for urls and paths
} else {
$hash = $combine;
}
return $hash;
}
/**
* Computes a keyed HMAC-SHA256 of a string for security-relevant
* hashing (CSRF tokens, signed cookies, etc.). Unlike crawlHash
* which is wired into binary index formats and must stay 8 bytes,
* this returns a full 32-byte HMAC keyed by AUTH_KEY so an
* attacker who cannot read AUTH_KEY cannot forge values. Use
* this rather than crawlHash anywhere a hash is being used as an
* authentication or authorization signal.
*
* @param string $string the string to hash
* @param bool $raw whether to leave raw (32 bytes) or base64-
* encode (43 chars URL-safe)
* @return string the HMAC-SHA256 of $string keyed by AUTH_KEY
*/
function crawlAuthHash($string, $raw = false)
{
$hmac = hash_hmac('sha256', $string, C\p('AUTH_KEY'), true);
return $raw ? $hmac : base64Hash($hmac);
}
/**
* Used to create a 20 byte hash of a string (typically a word or phrase
* with a wikipedia page). Format is 8 byte crawlHash of term (md5 of term
* two halves XOR'd), followed by a \x00, followed by the first 11 characters
* from the term. If there are not enough char's to make 20 bytes, then the
* string is padded with \x00s to 20bytes.
*
* @param string $string word to hash
* @param bool $raw whether to base64Hash the result
* @return string first 8 bytes of md5 of $string concatenated with \x00
* to indicate the hash is of a word not a phrase concatenated with the
* padded to 11 byte $meta_string.
*/
function crawlHashWord($string, $raw = false)
{
$pre_hash = substr(md5($string, true), 0, 8) .
"\x00" . substr($string, 0, 11);
$pre_hash = str_pad($pre_hash, 20, "\x00");
if (!$raw) {
$hash = base64Hash($pre_hash);
// common variant of base64 safe for urls and paths
} else {
$hash = $pre_hash;
}
return $hash;
}
/**
* Take a $term that might have come from adocuments and converts it to
* a string of 16 bytes which is either the original term padded by
* underscores or the first seven chars of the term followed by an underscore
* followed by the base64 encoding of the first 6 chars of its md5 hash.
* Base64 used to make this all nice and printable.
* @param string $term to made into a canonical form
* @return string canonicalize by apbove version of term.
*/
function canonicalTerm($term)
{
$term = encodeUnderscore($term);
$term = rawurlencode($term);
$term = str_replace("%", "", $term);
$hash = (strlen($term) < 16) ? str_pad($term, 16, "_") :
substr($term, 0, 7) . "_" . base64Hash(
substr(md5($term, true), 0, 6));
return $hash;
}
/**
* Used to compare to ids for index dictionary lookup. ids
* are a 8 byte crawlHash together with 12 byte non-hash suffix.
*
* @param string $id1 20 byte word id to compare
* @param string $id2 20 byte word id to compare
* @return int negative if $id1 smaller, positive if bigger, and 0 if
* same
*/
function compareWordHashes($id1, $id2)
{
return strcmp($id1, $id2);
}
/**
* Converts a crawl hash number to something closer to base64 coded but
* so doesn't get confused in urls or DBs
*
* @param string $string a hash to base64 encode
* @return string the encoded hash
*/
function base64Hash($string)
{
return strtr(rtrim(base64_encode($string), "="), ["/" => "_", "+" => "-"]);
}
/**
* Decodes a crawl hash number from base64 to raw ASCII
*
* @param string $base64 a hash to decode
* @return string the decoded hash
*/
function unbase64Hash($base64)
{
//get rid of out modified base64 encoding
return base64_decode(strtr($base64, ["_" => "/", "-" => "+"]) . "=");
}
/**
* Encodes a string in a format suitable for post data
* (mainly, base64, but str_replace data that might mess up post in result)
*
* @param string $str string to encode
* @return string encoded string
*/
function webencode($str)
{
return strtr(base64_encode($str), ["/" => "_", "=" => "~", "+" => "."]);
}
/**
* Decodes a string encoded by webencode
*
* @param string $str string to encode
* @return string encoded string
*/
function webdecode($str)
{
return base64_decode(strtr($str ?? "",
["." => "+", "~" => "=", "_" => "/"]));
}
/**
* The crawlHash function is used to encrypt passwords stored in the database.
* It tries to use the best version the Blowfish variant of php's crypt
* function available on the current system.
*
* @param string $string the string to encrypt
* @param int $salt salt value to be used (needed to verify if a password is
* valid)
* @return string the crypted string where crypting is done using crawlHash
*/
function crawlCrypt($string, $salt = null)
{
if ($salt === null) {
/* The length of the salt and its starting prefix say which hash
public function crypt uses. Blowfish's begins with $2a$, $2x$,
or $2y$ followed by the base 2 logarithm of the iteration count
for blowfish. We use the more secure 2y.
*/
$salt = '$2y$12$' . strtr(base64_encode(random_bytes(16)), '+', '.');
}
/* Run the hash here rather than handing it to a short-lived helper
process. The bcrypt cost is chosen so a single hash takes only a
fraction of a second, which is less than it costs to spawn a
separate PHP process, so on the single-process mail and web
servers computing it directly is both faster and simpler than
offloading it. */
return crypt($string, $salt);
}
/**
* Lists the ways a candidate password fails the site's configured
* password policy. The policy is set in the admin Security activity and
* applies to the passwords Yioop stores itself: a smallest allowed
* length, optional requirements that the password contain a lower-case
* letter, an upper-case letter, a digit, or a punctuation symbol, and a
* set of characters that are never allowed (by default the single and
* double quote). The result is a list of short token strings naming each
* unmet rule and is empty when the password is acceptable. A controller
* turns the tokens into messages a person reads, so this stays free of
* any presentation text.
*
* @param string $password candidate password to check
* @param array|null $policy the policy to check against, with keys
* min_length, max_length, lowercase, uppercase, digit, symbol, and
* forbidden; when null the site's configured policy is read from
* Config, with the require-flags coerced to real booleans so a
* profile that stored them as strings cannot read as always-on
* @return array unmet-rule tokens such as "too_short", "digit",
* "forbidden"; empty when the password satisfies every rule
*/
function passwordPolicyViolations($password, $policy = null)
{
if ($policy === null) {
$policy = [
"min_length" => C\p('PASSWORD_MIN_LEN'),
"max_length" => C\LONG_NAME_LEN,
"lowercase" => filter_var(C\p('PASSWORD_REQUIRE_LOWERCASE'),
FILTER_VALIDATE_BOOLEAN),
"uppercase" => filter_var(C\p('PASSWORD_REQUIRE_UPPERCASE'),
FILTER_VALIDATE_BOOLEAN),
"digit" => filter_var(C\p('PASSWORD_REQUIRE_DIGIT'),
FILTER_VALIDATE_BOOLEAN),
"symbol" => filter_var(C\p('PASSWORD_REQUIRE_SYMBOL'),
FILTER_VALIDATE_BOOLEAN),
"forbidden" => C\PASSWORD_FORBIDDEN_CHARS,
];
}
$violations = [];
$length = strlen($password);
if ($length < $policy["min_length"]) {
$violations[] = "too_short";
}
if ($length > $policy["max_length"]) {
$violations[] = "too_long";
}
if ($policy["lowercase"] && !preg_match('/[a-z]/', $password)) {
$violations[] = "lowercase";
}
if ($policy["uppercase"] && !preg_match('/[A-Z]/', $password)) {
$violations[] = "uppercase";
}
if ($policy["digit"] && !preg_match('/[0-9]/', $password)) {
$violations[] = "digit";
}
if ($policy["symbol"] &&
!preg_match('/[^a-zA-Z0-9]/', $password)) {
$violations[] = "symbol";
}
if ($policy["forbidden"] !== "" &&
strpbrk($password, $policy["forbidden"]) !== false) {
$violations[] = "forbidden";
}
return $violations;
}
/**
* Gives the cooperative event loop a brief turn in the middle of a long
* piece of work. Inside a fiber this suspends so the loop can serve other
* connections before resuming this one; outside a fiber (under Apache, or
* any caller that is not being run cooperatively) it does nothing, so the
* work runs straight through as before. A long loop calls this every so
* often so it does not freeze the single-process server while it runs.
*
* @return void
*/
function cooperativeYield()
{
if (\Fiber::getCurrent() !== null) {
\Fiber::suspend();
}
}
/**
* Used by a controller to take a table and return those rows in the
* table that a given queue_server would be responsible for handling
*
* @param array $table an array of rows of associative arrays which
* a queue_server might need to process
* @param string $field column of $table whose values should be used
* for partitioning
* @param int $num_partition number of queue_servers to choose between
* @param int $instance the id of the particular server we are interested
* in
* @param object $callback function or static method that might be
* applied to input before deciding the responsible queue_server.
* For example, if input was a url we might want to get the host
* before deciding on the queue_server
* @return array the reduced table that the $instance queue_server is
* responsible for
*/
function partitionByHash($table, $field, $num_partition, $instance,
$callback = null)
{
$out_table = [];
if (is_array($table)) {
foreach ($table as $row) {
$cell = ($field === null) ? $row : $row[$field];
$hash_int = calculatePartition($cell, $num_partition, $callback);
if ($hash_int == $instance) {
$out_table[] = $row;
}
}
}
return $out_table;
}
/**
* Used by a controller to say which queue_server should receive
* a given input
* @param string $input can view as a key that might be processes by a
* queue_server. For example, in some cases input might be
* a url and we want to determine which queue_server should be
* responsible for queuing that url
* @param int $num_partition number of queue_servers to choose between
* @param object $callback function or static method that might be
* applied to input before deciding the responsible queue_server.
* For example, if the input was a url we might want to get the host
* before deciding on the queue_server
* @return int id of server responsible for input
*/
function calculatePartition($input, $num_partition, $callback = null)
{
if ($callback !== null) {
$callback_parts = explode("::", $callback);
if (count($callback_parts) == 1) {
$input = $callback($input);
} else {
$class_name = $callback_parts[0];
$method_name = $callback_parts[1];
$tmp_class = new $class_name;
$input = $tmp_class->$method_name($input);
}
}
$hash_int = abs(unpackInt(substr(crawlHash($input, true), 0, 4))) %
$num_partition;
return $hash_int;
}
/**
* Measures the change in time in seconds between two timestamps to microsecond
* precision
*
* @param string $start starting time with microseconds
* @param string $end ending time with microseconds, if null use current time
* @return float time difference in seconds
*/
function changeInMicrotime($start, $end = null)
{
if ( !$end ) {
$end = microtime(true);
}
return $end - $start;
}
/**
* Timestamp of current epoch with microsecond precision useful for situations
* where time() might cause too many collisions (account creation, etc)
* @return string timestamp to microsecond of time in second since start of
* current epoch
*/
function microTimestamp()
{
return vsprintf('%d.%06d', gettimeofday());
}
/**
* Checks that a timestamp is within the time interval given by a
* start time (HH:mm) and a duration
*
* @param string $start_time string of the form (HH:mm)
* @param string $duration string containing an int in seconds
* @param int $time a Unix timestamp.
* @return int -1 if the time of day of $time is not within the given interval.
* Otherwise, the Unix timestamp at which the interval will be over for
* the same day as $time.
*/
function checkTimeInterval($start_time, $duration, $time = -1)
{
$duration = intval($duration);
if ($duration <= 0 ) {
return -1;
}
if (intval($time) < 0 ) {
$time = time();
}
$today = date('Y-m-d', $time);
$timezone_offset = date('P', $time);
$start_time = trim($start_time);
if (!preg_match("/\d\d\:\d\d/", $start_time)) {
$start_time = "00:00";
}
$start_timestamp = strtotime($today . "T" . $start_time . ":00".
$timezone_offset);
$end_timestamp = $start_timestamp + $duration;
if ($time >= $start_timestamp && $time <= $end_timestamp) {
return $end_timestamp;
}
return -1;
}
/**
* Converts a CSS unit string into its equivalent in pixels. This is
* used by @see SvgProcessor.
*
* @param string $value a number followed by a legal CSS unit
* @return int a number in pixels
*/
function convertPixels($value)
{
$len = strlen($value);
if (is_int($value) || $len < 2) {
return intval($value);
}
if ($value[$len - 1] == "%") {
$num = floatval(substr($value, 0, $len - 1));
return ($num > 0) ? floor(8 * min(100, $num)) : 0;
}
$num = floatval(substr($value, 0, $len - 2));
$unit = substr($value, $len - 2);
return match ($unit) {
"cm", "pt" => intval(28 * $num),
"em", "pc" => intval(6 * $num),
"ex" => intval(12 * $num),
"in" => intval(72 * $num), //assume screen 72 dpi as on mac
"mm" => intval(2.8 * $num),
"px" => intval($num),
default => intval($value),
};
}
/**
* Returns the number of files in a folder
* @param string $folder path to folder to count
* @return int number of files
*/
function countFiles($folder)
{
$file_iterator = new \FilesystemIterator($folder);
return iterator_count($file_iterator);
}
/**
* Creates folders along a filesystem path if they don't exist
*
* @param string $path a file system path
* @return bool success or failure
*/
function makePath($path)
{
$path_parts = explode("/", $path);
$num_parts = count($path_parts);
$path_so_far = "";
for ($i = 0; $i < $num_parts; $i++) {
$path_so_far .= $path_parts[$i] . "/";
if (!is_dir($path_so_far)) {
if (file_exists($path_so_far) || !mkdir($path_so_far) ) {
return false;
}
}
}
return true;
}
/**
* This is a callback function used in the process of recursively deleting a
* directory
*
* @param string $file_or_dir the filename or directory name to be deleted
* @see DatasourceManager::unlinkRecursive()
*/
function deleteFileOrDir($file_or_dir)
{
if (file_exists($file_or_dir)) {
if (is_file($file_or_dir)) {
unlink($file_or_dir);
} else {
rmdir($file_or_dir);
}
}
}
/**
* This is a callback function used in the process of recursively chmoding to
* 777 all files in a folder
*
* @param string $file the filename or directory name to be chmod
* @see DatasourceManager::setWorldPermissionsRecursive()
*/
function setWorldPermissions($file)
{
if (php_sapi_name() == 'cli') {
chmod($file, 0777);
return;
}
set_error_handler(null);
chmod($file, 0777);
restore_error_handler();
}
/**
* This is a callback function used in the process of recursively calculating
* an array of file modification times and files sizes for a directory
*
* @param string $file a name of a file in the file system
* @return an array whose single element contain an associative array
* with the size and modification time of the file
*/
function fileInfo($file)
{
$info["name"] = $file;
$info["size"] = filesize($file);
$info["is_dir"] = is_dir($file);
$info["modified"] = filemtime($file);
return [$info];
}
//ordering functions used in sorting
/**
* Callback function used to sort documents by a field
*
* Should be initialized before using in usort with a call
* like: orderCallback($tmp, $tmp, "field_want");
*
* @param string $word_doc_a doc id of first document to compare
* @param string $word_doc_b doc id of second document to compare
* @param string $order_field which field of these associative arrays to sort by
* @return int -1 if first doc bigger 1 otherwise
*/
function orderCallback($word_doc_a, $word_doc_b, $order_field = null)
{
static $field = "a";
if ($order_field !== null) {
$field = $order_field;
return -1;
}
return ((float)$word_doc_a[$field] >
(float)$word_doc_b[$field]) ? -1 : 1;
}
/**
* Callback function used to sort documents by a field where field is assume to
* be a string
*
* Should be initialized before using in usort with a call
* like: stringOrderCallback($tmp, $tmp, "field_want");
*
* @param string $word_doc_a doc id of first document to compare
* @param string $word_doc_b doc id of second document to compare
* @param string $order_field which field of these associative arrays to sort by
* @return int -1 if first doc smaller 1 otherwise
*/
function stringOrderCallback($word_doc_a, $word_doc_b, $order_field = null)
{
static $field = "a";
if ($order_field !== null) {
$field = $order_field;
return -1;
}
return ((string)$word_doc_a[$field] >
(string)$word_doc_b[$field]) ? -1 : 1;
}
/**
* Callback function used to sort documents by a field where field is assume to
* be a string
*
* Should be initialized before using in usort with a call
* like: stringROrderCallback($tmp, $tmp, "field_want");
*
* @param string $word_doc_a doc id of first document to compare
* @param string $word_doc_b doc id of second document to compare
* @param string $order_field which field of these associative arrays to sort by
* @return int -1 if first doc bigger 1 otherwise
*/
function stringROrderCallback($word_doc_a, $word_doc_b, $order_field = null)
{
static $field = "a";
if ($order_field !== null) {
$field = $order_field;
return -1;
}
return ((string)$word_doc_a[$field] <
(string)$word_doc_b[$field]) ? -1 : 1;
}
/**
* Callback function used to sort documents by a field in reverse order
*
* Should be initialized before using in usort with a call
* like: rorderCallback($tmp, $tmp, "field_want");
*
* @param string $word_doc_a doc id of first document to compare
* @param string $word_doc_b doc id of second document to compare
* @param string $order_field which field of these associative arrays to sort by
* @return int 1 if first doc bigger -1 otherwise
*/
function rorderCallback($word_doc_a, $word_doc_b, $order_field = null)
{
static $field = "a";
if ($order_field !== null) {
$field = $order_field;
return -1;
}
return ((float)$word_doc_a[$field] >
(float)$word_doc_b[$field]) ? 1 : -1;
}
/**
* Callback to check if $a is less than $b
*
* Used to help sort document results returned in PhraseModel called
* in IndexArchiveBundle
*
* @param float $a first value to compare
* @param float $b second value to compare
* @return int -1 if $a is less than $b; 1 otherwise
* @see IndexArchiveBundle::getSelectiveWords()
* @see PhraseModel::getPhrasePageResults()
*/
function lessThan($a, $b) {
if ($a == $b) {
return 0;
}
return ($a < $b) ? -1 : 1;
}
/**
* Callback to check if $a is greater than $b
*
* Used to help sort document results returned in PhraseModel called in
* IndexArchiveBundle
*
* @param float $a first value to compare
* @param float $b second value to compare
* @return int -1 if $a is greater than $b; 1 otherwise
* @see IndexArchiveBundle::getSelectiveWords()
* @see PhraseModel::getTopPhrases()
*/
function greaterThan($a, $b) {
if ($a == $b) {
return 0;
}
return ($a > $b) ? -1 : 1;
}
/**
* shorthand for echo
*
* @param string $text string to send to the current output
*/
function e($text)
{
echo $text;
}
/**
* Compute the real remote address of the incoming connection
* including forwarding
*
* @return string the client IP address: HTTP_X_FORWARDED_FOR if set,
* otherwise REMOTE_ADDR, falling back to "127.0.0.1"
*/
function remoteAddress()
{
return (!empty($_SERVER['HTTP_X_FORWARDED_FOR'])) ?
$_SERVER['HTTP_X_FORWARDED_FOR'] : (
(!empty($_SERVER['REMOTE_ADDR'])) ? $_SERVER['REMOTE_ADDR'] :
"127.0.0.1");
}
/**
* Used to read a line of input from the command-line
* @return string from the command-line
*/
function readInput()
{
$stdin = fopen('php://stdin', 'r');
$line = fgets($stdin);
$line = trim($line);
fclose($stdin);
return $line;
}
/**
* Used to read a line of input from the command-line
* (on unix machines without echoing it)
* @return string from the command-line
*/
function readPassword()
{
system('stty -echo');
$line = readInput();
if (!strstr(PHP_OS, "WIN")) {
e(str_repeat("*", strlen($line))."\n");
}
system('stty echo');
return $line;
}
/**
* Used to read a several lines from the terminal up until
* a last line consisting of just a "."
* @return string from the command-line
*/
function readMessage()
{
$message = "";
$line = "";
do {
$message .= $line;
$line = readInput()."\n";
} while(rtrim($line) != ".");
return rtrim($message);
}
/**
* Returns the mime type of the provided file name if it can be determined.
*
* @param string $file_name (name of file including path to figure out
* mime type for)
* @param bool $use_extension whether to just try to guess from the file
* extension rather than looking at the file
* @return string mime type or unknown if can't be determined
*/
function mimeType($file_name, $use_extension = false)
{
return A\WebSite::mimeType($file_name, $use_extension);
}
/**
* Checks if class_1 is the same as class_2 or has class_2 as a parent
* Behaves like 3 param version (last param true) of PHP is_a function
* that came into being with Version 5.3.9.
*
* @param mixed $class_1 object or string class name to see if in class2
* @param mixed $class_2 object or string class name to see if contains class1
* @return bool equal or contains class
*/
function generalIsA($class_1, $class_2)
{
if ($class_1 == $class_2) {
return true;
}
return (is_a($class_1, $class_2) || is_subclass_of($class_1, $class_2));
}
/**
* Given the contents of a start XML/HMTL tag strips out all the attributes
* non listed in $safe_attribute_list
*
* @param string $start_tag_contents the contents of an HTML/XML tag. I.e.,
* if the tag was <tag stuff> then $start_tag_contents could be stuff
* @param array $safe_attribute_list a list of attributes which should be kept
* @return string containing only safe attributes and their values
*/
function stripAttributes($start_tag_contents, $safe_attribute_list = [])
{
$out = "";
if ($safe_attribute_list != []) {
$safe_regex = '/(?:(?:\A|\s+)(?:';
$first = "";
foreach ($safe_attribute_list as $attribute) {
$safe_regex .= $first . $attribute;
$first = "|";
}
$safe_regex .= ')\s*=\s*(?:"[^"]+"|'."'[^']+'))/";
preg_match_all($safe_regex, $start_tag_contents, $matches);
if (isset($matches[0])) {
foreach ($matches[0] as $attribute) {
$out .= " ".$attribute;
}
if ($out) {
$out = trim($out);
}
}
}
return $out;
}
/**
* Used to parse into a two dimensional array a string that contains CSV data.
* @param string $pre_csv_string string with csv data
* @return array two dimensional array of elements from csv
*/
function parseCsv($pre_csv_string)
{
$len = strlen($pre_csv_string);
$csv_string = "";
$quote = false;
$escape = false;
$slash = chr(92);
$slashn = $slash . "n";
$slashr = $slash . "r";
for ($i = 0; $i < $len; $i++) {
$char = ord($pre_csv_string[$i]);
$escape = (!$escape && $char == 92) ? true : false;
if (!$escape && $char == 34) {
$quote = ($quote) ? false : true;
}
$out_char = chr($char);
if ($quote) {
if ($char == 10) {
$out_char = $slashn;
} else if ($char == 13) {
$out_char = $slashr;
}
}
$csv_string .= $out_char;
}
$lines = str_getcsv($csv_string, "\n", escape: "");
$out_lines = [];
foreach ($lines as $line) {
$line = str_replace($slashr, "\r", $line);
$line = str_replace($slashn, "\n", $line);
$out_lines[] = str_getcsv($line, escape: "");
}
return $out_lines;
}
/**
* Converts an array of values to a comma separated value formatted string.
*
* @param array $arr values to convert
* @return string CSV string after conversion
*/
function arraytoCsv($arr)
{
$fp = fopen('php://temp', 'r+');
fputcsv($fp, $arr, escape: "\\");
rewind($fp);
$data = fgets($fp);
fclose($fp);
return rtrim($data, "\n");
}
/**
* Computes a Unix-style diff of two strings. That is it only
* outputs lines which disagree between the two strings. It outputs +line
* if a line occurs in the second but not first string and -line if a
* line occurs in the first string but not the second.
*
* When the two strings differ so extensively that an exact line-by-line
* diff would build a longest-common-subsequence table larger than
* MAX_DIFF_LCS_CELLS, the differing region is reported coarsely instead:
* one block of removed lines followed by one block of added lines. This
* keeps the memory used by a single diff bounded no matter how large the
* two versions are.
*
* @param string $data1 first string to compare
* @param string $data2 second string to compare
* @param bool $html whether to output html highlighting
* @return string representing info about where $data1 and $data2 don't match
*/
function diff($data1, $data2, $html = false)
{
if ($html) {
$start = "<div>";
$start_same = "<div class='light-gray'>";
$start1 = "<div class='red'>";
$start2 = "<div class='green'>";
$end = "</div>";
} else {
$start = "";
$start_same = "";
$start1 = "";
$start2 = "";
$end = "";
}
$lines1 = explode("\n", $data1);
$lines2 = explode("\n", $data2);
$num_lines1 = count($lines1);
$num_lines2 = count($lines2);
$shorter_len = min($num_lines1, $num_lines2);
$longer_len = max($num_lines1, $num_lines2);
$first_diff = 0;
// trim off the starts and end lines of the strings that are same
$head_lcs = [];
while($first_diff < $shorter_len &&
strcmp($lines1[$first_diff], $lines2[$first_diff]) == 0) {
$head_lcs[] = [$first_diff, $first_diff, $lines1[$first_diff]];
$first_diff++;
}
if ($first_diff == $shorter_len) {
if ($num_lines1 == $num_lines2) {
return "";
} else {
$tmp = $lines1;
$prefix = "$start1-";
if ($num_lines1 == $shorter_len) {
$tmp = $lines2;
$prefix = "$start2+";
}
$out = "$start@@ -$shorter_len,0 +$shorter_len,".
"$longer_len @@$end\n";
for ($i = $shorter_len; $i < $longer_len; $i++) {
$out .= $prefix . $tmp[$i]."$end\n";
}
return $out;
}
}
$last_diff = 0;
$tail_lcs = [];
$index1 = $num_lines1 - 1 - $last_diff;
$index2 = $num_lines2 - 1 - $last_diff;
while($shorter_len - $last_diff > $first_diff &&
$lines1[$index1] == $lines2[$index2]) {
array_unshift($tail_lcs, [$index1, $index2, $lines1[$index1]]);
$last_diff++;
$index1--;
$index2--;
}
$trim_lines1 = array_slice($lines1, $first_diff,
$num_lines1 - $first_diff - $last_diff);
$trim_lines2 = array_slice($lines2, $first_diff,
$num_lines2 - $first_diff - $last_diff);
/* To compute a diff, we first compute the
LCS = Longest common subsequence of the two string. The table this
needs has one cell per pair of differing lines, so on two versions
that disagree throughout a large page it can grow to hundreds of
megabytes for only a few kilobytes of visible difference. Past a
safe ceiling we skip the table and leave $lcs empty, which makes
the code below report the whole differing region coarsely as one
removed block followed by one added block at almost no memory cost.
*/
$lcs_cells = count($trim_lines1) * count($trim_lines2);
if ($lcs_cells > C\MAX_DIFF_LCS_CELLS) {
$lcs = [];
} else {
$lcs = computeLCS($trim_lines1, $trim_lines2, $first_diff);
}
$lcs = array_merge($head_lcs, $lcs, $tail_lcs);
$previous_first = -1;
$previous_second = -1;
$current_first = 0;
$current_second = 0;
$old_line = "";
$out_string = "";
if ($lcs == []) {
$out_string .= "$start@@ -0,$num_lines1 ".
" +0,$num_lines2 @@$end\n";
for ($i = 0; $i < $num_lines1; $i++) {
$out_string .= "$start1-" . $lines1[$i] . "$end\n";
}
for ($i = 0; $i < $num_lines2; $i++) {
$out_string .= "$start2+" . $lines2[$i] . "$end\n";
}
} else {
foreach ($lcs as $lcs_item) {
list($current_first, $current_second, $line) = $lcs_item;
$gap1 = $current_first - $previous_first;
$gap2 = $current_second - $previous_second;
if ($gap1 > 1 || $gap2 > 1) {
$gap1++;
$gap2++;
$out_string .= "$start@@ -$previous_first,$gap1 ".
" +$previous_second,$gap2 @@$end\n";
$out_string .= "$start_same ".$old_line."$end\n";
for ($i = $previous_first + 1; $i < $current_first; $i++) {
$out_string .= "$start1-" . $lines1[$i] . "$end\n";
}
for ($i = $previous_second + 1; $i < $current_second; $i++) {
$out_string .= "$start2+" . $lines2[$i] . "$end\n";
}
$out_string .= "$start_same ".$line."$end\n";
}
$previous_first = $current_first;
$previous_second = $current_second;
$old_line = $line;
}
if ($current_first < $num_lines1 - 1 ||
$current_second < $num_lines2 - 1) {
$gap1 = $num_lines1 - $current_first;
$gap2 = $num_lines2 - $current_second;
$out_string .= "$start_same ".$line."$end\n";
for ($i = $current_first + 1; $i < $num_lines1; $i++) {
$out_string .= "$start1-" . $lines1[$i] . "$end\n";
}
for ($i = $current_second + 1; $i < $num_lines2; $i++) {
$out_string .= "$start2+" . $lines2[$i] . "$end\n";
}
}
}
return $out_string;
}
/**
* Computes the longest common subsequence of two arrays
*
* @param array $lines1 an array of lines to compute LCS of
* @param array $lines2 an array of lines to compute LCS of
* @param int $offset an offset to shift over array addresses in output by
* @return array list of triples (index_string1, index_string2, line)
* describing the longest common subsequence
*/
function computeLCS($lines1, $lines2, $offset = 0)
{
/*
add a dummy line so don't have to worry about
shifting indices in CLRS pseudo-code implementation
*/
$num_lines1 = count($lines1);
$num_lines2 = count($lines2);
array_unshift($lines1, 0);
array_unshift($lines2, 0);
/*
LCS = Longest common subsequence of the two string.
The code below is based off the pseudo-code in CLRS
*/
$lcs_moves = [];
$lcs_values = [];
for ($i = 1; $i <= $num_lines1; $i++) { //initialize first column
$lcs_values[$i][0] = 0;
}
for ($j = 0; $j <= $num_lines2; $j++) { //initialize first column
$lcs_values[0][$j] = 0;
}
$lcs_moves = [];
for ($i = 1; $i <= $num_lines1; $i++) {
for ($j = 1; $j <= $num_lines2; $j++) {
if ($lines1[$i] == $lines2[$j]) {
$lcs_values[$i][$j] = $lcs_values[$i - 1][$j - 1] + 1;
$lcs_moves[$i][$j] = "d"; //diagonal
} elseif ($lcs_values[$i - 1][$j] >= $lcs_values[$i][$j - 1]) {
$lcs_values[$i][$j] = $lcs_values[$i - 1][$j];
$lcs_moves[$i][$j] = "u"; // up
} else {
$lcs_values[$i][$j] = $lcs_values[$i][$j - 1];
$lcs_moves[$i][$j] = "l"; // left
}
}
}
$lcs = [];
extractLCSFromTable($lcs_moves, $lines1, $num_lines1, $num_lines2,
$offset, $lcs);
return $lcs;
}
/**
* Extracts from a table of longest common sequence moves (probably calculated
* by @see computeLCS) and a starting coordinate $i, $j in that table,
* a longest common subsequence
*
* @param array $lcs_moves a table of move computed by computeLCS
* @param array $lines from first of the two arrays computing LCS of
* @param int $i a line number in string 1
* @param int $j a line number in string 2
* @param int $offset a number to add to each line number output into $lcs.
* This is useful if we have trimmed off the initially common lines from
* our two strings we are trying to compute the LCS of
* @param array &$lcs an array of triples
* (index_string1, index_string2, line)
* the indexes indicate the line number in each string, line is the line
* in common the two strings
* @return array empty list when $i or $j is zero (the recursion base
* case); otherwise the in-progress $lcs is mutated by reference
*/
function extractLCSFromTable($lcs_moves, $lines, $i, $j, $offset, &$lcs)
{
if ($i == 0 || $j == 0) {
return [];
}
if ($lcs_moves[$i][$j] == "d") { //diagonal moves means common to both
//sub-case first so forward order
extractLCSFromTable($lcs_moves, $lines, $i - 1, $j - 1, $offset, $lcs);
$lcs[] = [$i + $offset- 1, $j + $offset - 1, $lines[$i]];
} elseif ($lcs_moves[$i][$j] == "u") { // up move in matrix
extractLCSFromTable($lcs_moves, $lines, $i - 1, $j, $offset, $lcs);
} else { // left move in matrix
extractLCSFromTable($lcs_moves, $lines, $i, $j - 1, $offset, $lcs);
}
}
/**
* Returns an array of the last $num_lines many lines our of a file
*
* @param string $file_name name of file to return lines from
* @param string $num_lines number of lines to retrieve
* @return array retrieved lines
*/
function tail($file_name, $num_lines)
{
$size = filesize($file_name);
$max_line_len = 160;
$offset = max(0, $size - $max_line_len * $num_lines);
$file_string = file_get_contents($file_name, $offset);
$lines = explode("\n", $file_string);
$lines = array_slice($lines, -$num_lines);
return $lines;
}
/**
* Given an array of lines returns a subarray of those lines containing the
* filter string or filter array
*
* @param string $lines to search
* @param mixed $filters either string to filter lines with or an array of
* strings (any of which can be present to pass the filter)
* @param bool $case_insensitive whether search should be done case
* insensitively or not.
* @return array lines containing the string
*/
function lineFilter($lines, $filters, $case_insensitive = true)
{
$out_lines = [];
$search_function = ($case_insensitive) ? "stripos" : "strpos";
if (is_string($filters)) {
$filters = [$filters];
}
foreach ($lines as $line) {
foreach ($filters as $filter) {
if ($search_function($line, $filter) !== false) {
$out_lines[] = $line;
break;
}
}
}
return $out_lines;
}
/**
* Tries to extract a timestamp from a line which is presumed to come from
* a Yioop log file
*
* @param string $line to search
* @return int timestamp of that log entry
*/
function logLineTimestamp($line)
{
preg_match('/^\s*\[\s*\d+\s+(.*)\]/', $line, $matches);
if (isset($matches[1])) {
return @strtotime($matches[1]);
}
return 0;
}
/**
* Returns whether an input can be parsed to a positive integer
*
* @param mixed $input value to test; treated as a positive integer
* when it is an int > 0 or a string of digits whose int value
* is > 0
* @return bool whether $input can be parsed to a positive integer.
*/
function isPositiveInteger($input)
{
return (is_int($input) && $input > 0) ||
(is_string($input) &&
preg_match("/^\d+$/", trim($input)) && intval($input) > 0);
}
/**
* Used to measure the memory footprint in bytes and time spent calling a method
* of an object. It also records number of time the method has been called.
* Just calls the method without any recording or timing until
* an initial call to the function measureCall(null, save_statistics_file)
* where save_statistics_file is the name of the file you won't to store
* statistics to.
*
* @param object $object name of object whose method we want to call and measure
* @param string $method method we're calling
* @param array $arguments arguments to be sent to the method
* @param string $call_name name to use when outputting stats for this call,
* defaults to $method.
* @return mixed whatever method would normally returned when called as above
*/
function measureCall($object, $method, $arguments = [], $call_name = "")
{
return measureObjectCall($object, $method, $arguments, $call_name);
}
/**
* Used to measure the memory footprint of an object in Yioop and save
* it to a statistics file
* No recording is done until
* an initial call to the function measureCall(null, save_statistics_file)
* where save_statistics_file is the name of the file you won't to store
* statistics to.
* @param object $object name of object whose size we want to measure
* @param string $save_file statistics file to write info to
* @param string $class_name name of class to use for measuring. If empty
* will call getClass to determine this
* @return mixed whatever measureObjectCall returns; on the initialization
* call (object null, save_file set) this is the boolean indicating
* whether stats recording is now active
*/
function measureObject($object, $save_file = "", $class_name = "")
{
return measureObjectCall($object, $save_file, [], $class_name);
}
/**
* General method called by for @see measureCall and @see measureObject
* Used to measure the memory footprint in bytes of an object or memory and
* time spent calling a method of an object. It also records number of time
* the method has been called. When used to call a method before initialization,
* just calls the method without any recording or timing. To initialize,
* an initial call to the function measureCall(null, save_statistics_file)
* where save_statistics_file is the name of the file you won't to store
* statistics to should be done.
*
* @param object $object name of object whose method we want to call and measure
* @param string $method method we're calling
* @param array $arguments arguments to be sent to the method
* @param string $call_name name to use when outputting stats for this call,
* defaults to $method.
* @return mixed whatever method would normally returned when called as above
*/
function measureObjectCall($object, $method, $arguments = [], $call_name = "")
{
static $record_file = "";
static $statistics = ["LAST_UPDATE" => 0, "METHODS" => [], "OBJECTS" => []];
if ($object == null && !empty($method)) {
$record_file = $method;
return true;
}
if (empty($method)) {
if (empty($record_file) || (!is_object($object) && !is_array($object)
&& empty($call_name))) {
return false;
}
if (!empty($call_name)) {
$object_name = $call_name;
} else {
$object_name = get_class($object);
}
if (!is_object($object) && !is_array($object)) {
$object_stats = strlen(serialize($object));
} else {
$object_stats = [];
foreach ($object as $elt => $value) {
$value_memory = strlen(serialize($value));
$object_stats[$elt] = $value_memory;
}
ksort($object_stats);
}
$statistics["OBJECTS"][$object_name] = $object_stats;
$result = true;
} else {
if (empty($record_file)) {
return call_user_func_array([$object, $method], $arguments);
}
if (empty($call_name)) {
$call_name = $method;
}
$method_stat = $statistics["METHODS"][$call_name] ??
["CALL_COUNT" => 0, "TIME_SPENT" => 0, "MEMORY_NOT_RETURNED" => 0];
$before_memory = memory_get_usage();
$start_time = microtime(true);
$result = call_user_func_array([$object, $method], $arguments);
garbageCollect();
$after_memory = memory_get_usage();
$result_clone = variableClone($result);
$delta_clone_memory = memory_get_usage() - $after_memory;
unset($result_clone);
garbageCollect();
$delta_memory = $after_memory - $delta_clone_memory - $before_memory;
$method_stat["MEMORY_NOT_RETURNED"] += $delta_memory;
$method_stat["TIME_SPENT"] = changeInMicrotime($start_time);
$method_stat["CALL_COUNT"]++;
$statistics["METHODS"][$call_name] = $method_stat;
}
if (time() - $statistics["LAST_UPDATE"] > C\ONE_MINUTE) {
$statistics["LAST_UPDATE"] = time();
file_put_contents($record_file, print_r($statistics, true));
}
return $result;
}
/**
* Makes a deep copy of a variable regardless of its type
* @param mixed $var variable to deep copy
* @return mixed the deep copy
*/
function variableClone($var)
{
if (is_array($var)) {
return array_map(C\NS_LIB . "variableClone", $var);
} else if (is_object($var)) {
return clone $var;
} else {
return $var;
}
}
/**
* Runs various system garbage collection functions and returns
* number of bytes freed.
*
* @return int number of bytes freed
*/
function garbageCollect()
{
$bytes_collected = 0;
if (function_exists("gc_mem_caches")) {
$bytes_collected += gc_mem_caches();
}
$bytes_collected += gc_collect_cycles();
return $bytes_collected;
}
/**
* The dom method saveHTML has a tendency to replace UTF-8, non-ascii characters
* with html entities. This is supposed to save avoiding the replacement.
* What it does is to first save the dom, then it replaces htmlentities of the
* form &single_char; or &#some_number; with the UTF-8 they correspond to.
* It leaves all other entities as they are
*
* @param DOMDocument $dom DOM tree to serialize; not modified
* @return string output of saving html
*/
function utf8SafeSaveHtml(\DOMDocument $dom)
{
$out = $dom->saveHTML();
$out = preg_replace("/\&([a-zA-z][a-zA-z]+)\;/u", '-a-m-p-${1};', $out);
$out = html_entity_decode($out, ENT_QUOTES | ENT_HTML5, 'UTF-8');
$out = preg_replace("/\-a\-m\-p\-([a-zA-z][a-zA-z]+)\;/u", '&${1};', $out);
return $out;
}
/**
* A UTF-8 safe version of PHP's wordwrap function that wraps a string to a
* given number of characters
*
* @param string $string the input string
* @param int $width the number of characters at which the string will be
* wrapped
* @param string $break string used to break a line into two
* @param bool $cut whether to always force wrap at $width characters
* even if word hasn't ended
* @return string the given string wrapped at the specified length
*/
function utf8WordWrap($string, $width = 75, $break = "\n", $cut = false)
{
if($cut) {
/* Match anything 1 to $width chars long followed by whitespace or EOS,
otherwise match anything $width chars long
*/
$search = '/(.{1,'. $width .'})(?:\s|$)|(.{'.$width.'})/uS';
$replace = '$1$2' . $break;
} else {
/* Anchor the beginning of the pattern with a lookahead
to avoid crazy backtracking when words are longer than $width
*/
$search = '/(?=\s)(.{1,' . $width . '})(?:\s|$)/uS';
$replace = '$1'. $break;
}
return preg_replace($search, $replace, $string);
}
(c) 2026 Seekquarry - Open Source Search Engine Software