<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\locale\hi\resources;
use seekquarry\yioop\configs as C;
/**
* Hindi specific tokenization code. In particular, it has a stemmer,
* The stemmer is my stab at porting Ljiljana Dolamic (University of Neuchatel,
* www.unine.ch/info/clef/) Java stemming algorithm:
* http://members.unine.ch/jacques.savoy/clef/HindiStemmerLight.java.txt
* Here given a word, its stem is that part of the word that
* is common to all its inflected variants. For example,
* tall is common to tall, taller, tallest. A stemmer takes
* a word and tries to produce its stem.
*
* @author Chris Pollett
*/
class Tokenizer
{
/**
* A list of frequently occurring terms for this locale which should
* be excluded from certain kinds of queries. This is also used
* for language detection
* @var array
*/
public static $stop_words = ['рдЬреИрд╕рд╛', 'рдореИрдВ', 'рдЙрд╕рдХреЗ', 'рдХрд┐', 'рд╡рд╣', 'рдерд╛', 'рдХреЗ',
'рд▓рд┐рдП', 'рдкрд░', 'рд╣реИрдВ', 'рд╕рд╛рде', 'рд╡реЗ', 'рд╣реЛ', 'рдкрд░', 'рдПрдХ', 'рд╣реИ', 'рдЗрд╕', 'рд╕реЗ', 'рджреНрд╡рд╛рд░рд╛',
'рдЧрд░рдо', 'рд╢рдмреНрдж', 'рд▓реЗрдХрд┐рди', 'рдХреНрдпрд╛', 'рдХреБрдЫ', 'рд╣реИ', 'рдпрд╣', 'рдЖрдк', 'рдпрд╛', 'рдерд╛', 'рдХреА',
'рддрдХ', 'рдФрд░', 'рдПрдХ', 'рдореЗрдВ', 'рд╣рдо', 'рдХрд░', 'рд╕рдХрддреЗ', 'рд╣реИрдВ', 'рдмрд╛рд╣рд░', 'рдЕрдиреНрдп', 'рдереЗ', 'рдЬреЛ',
'рдХрд░', 'рдЙрдирдХреЗ', 'рд╕рдордп', 'рдЕрдЧрд░', 'рд╣реЛрдЧрд╛', 'рдХреИрд╕реЗ', 'рдХрд╣рд╛', 'рдПрдХ', 'рдкреНрд░рддреНрдпреЗрдХ', 'рдмрддрд╛',
'рдХрд░рддрд╛', 'рд╣реИ', 'рд╕реЗрдЯ', 'рддреАрди', 'рдЪрд╛рд╣рддреЗ рд╣реИрдВ', 'рд╣рд╡рд╛', 'рдЕрдЪреНрдЫреА рддрд░рд╣ рд╕реЗ', 'рднреА', 'рдЦреЗрд▓рдиреЗ',
'рдЫреЛрдЯреЗ', 'рдЕрдВрдд', 'рдбрд╛рд▓', 'рдШрд░', 'рдкрдврд╝рд╛', 'рд╣рд╛рде', 'рдмрдВрджрд░рдЧрд╛рд╣', 'рдмрдбрд╝рд╛', 'рдЬрд╛рджреВ', 'рдЬреЛрдбрд╝',
'рдФрд░', 'рднреА', 'рднреВрдорд┐', 'рдпрд╣рд╛рдБ', 'рдЪрд╛рд╣рд┐рдП', 'рдмрдбрд╝рд╛', 'рдЙрдЪреНрдЪ', 'рдРрд╕рд╛', 'рдХрд╛', 'рдкрд╛рд▓рди', 'рдХрд░реЗрдВ',
'рдЕрдзрд┐рдирд┐рдпрдо', 'рдХреНрдпреЛрдВ', 'рдкреВрдЫрдирд╛', 'рдкреБрд░реБрд╖реЛрдВ', 'рдкрд░рд┐рд╡рд░реНрддрди', 'рдЪрд▓рд╛', 'рдЧрдпрд╛', 'рдкреНрд░рдХрд╛рд╢', 'рддрд░рд╣',
'рдмрдВрдж', 'рдЖрд╡рд╢реНрдпрдХрддрд╛', 'рдШрд░', 'рддрд╕реНрд╡реАрд░', 'рдХреЛрд╢рд┐рд╢', 'рд╣рдореЗрдВ', 'рдлрд┐рд░', 'рдкрд╢реБ', 'рдмрд┐рдВрджреБ', 'рдорд╛рдВ',
'рджреБрдирд┐рдпрд╛', 'рдирд┐рдХрдЯ', 'рдмрдирд╛рдирд╛', 'рдЖрддреНрдо', 'рдкреГрдереНрд╡реА', 'рдкрд┐рддрд╛'];
/**
* List of verb-like parts of speech that might appear in lexicon
* @var array
*/
public static $verb_type = ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ",
"RB"];
/**
* List of noun-like parts of speech that might appear in lexicon
* @var array
*/
public static $noun_type = ["NN", "NNS", "NNP", "NNPS", "DT"];
/**
* List of adjective-like parts of speech that might appear in lexicon
* @var array
*/
public static $adjective_type = ["JJ", "JJR", "JJS"];
/**
* List of postpositional-like parts of speech that might appear in lexicon
* @var array
*/
public static $postpositional_type = ["IN", "inj", "PREP", "proNN",
"CONJ", "INT", "particle", "case", "PSP", "direct_DT", "PRP"];
/**
* List of questions in Hindi
* @var array
*/
public static $question_pattern =
"/\b[рдХреНрдпрд╛|рдХрдм|рдХрд╣рд╛|рдХреНрдпреЛрдВ|рдХреМрди|рдЬрд┐рд╕реЗ|рдЬрд┐рд╕рдХрд╛|рдХрд╣рд╛рдБ|рдХрд╣рд╛рдВ]\b/ui";
/**
* Any unique identifier corresponding to the component of a triplet which
* can be answered using a question answer list
* @var string
*/
public static $question_token = "qqq";
/**
* Words we don't want to be stemmed
* @var array
*/
public static $no_stem_list = [];
/**
* Removes the stop words from the page (used for Word Cloud generation
* and language detection)
*
* @param mixed $data either a string or an array of string to remove
* stop words from
* @return mixed $data with no stop words
*/
public static function stopwordsRemover($data)
{
static $pattern = "";
if (empty($pattern)) {
$pattern = '/\b(' . implode('|', self::$stop_words) . ')\b/u';
}
$data = preg_replace($pattern, '', $data);
return $data;
}
/**
* Stub function which could be used for a word segmenter.
* Such a segmenter on input thisisabunchofwords would output
* this is a bunch of words
*
* @param string $pre_segment before segmentation
* @return string should return string with words separated by space
* in this case does nothing
*/
public static function segment($pre_segment)
{
return $pre_segment;
}
/**
* Computes the stem of an Hindi word
*
* @param string $word the string to stem
* @return string the stem of $word
*/
public static function stem($word)
{
return $word;
}
/**
* Removes common Hindi suffixes
*
* @param string $word to remove suffixes from
* @return string result of suffix removal
*/
private static function removeSuffix($word)
{
return $word;
}
/**
* The method takes as input a phrase and returns a string with each
* term tagged with a part of speech.
*
* @param string $phrase text to add parts speech tags to
* @param bool $with_tokens whether to include the terms and the tags
* in the output string or just the part of speech tags
* @return string $tagged_phrase which is a string of format term~pos
*/
public static function tagPartsOfSpeechPhrase($phrase, $with_tokens = true)
{
$tagged_tokens = self::tagTokenizePartOfSpeech($phrase);
$tagged_phrase = self::taggedPartOfSpeechTokensToString(
$tagged_tokens, $with_tokens);
return $tagged_phrase;
}
/**
* Uses the lexicon to assign a tag to each token and then uses a rule
* based approach to assign the most likely of tags to each token
*
* @param string $text input phrase which is to be tagged
* @return string $result which is an array of token => tag
*/
public static function tagTokenizePartOfSpeech($text)
{
static $dictionary = [];
$lexicon_file = C\LOCALE_DIR . "/hi/resources/lexicon.txt.gz";
if (empty($dictionary)) {
if (file_exists($lexicon_file)) {
$lex_data = gzdecode(file_get_contents($lexicon_file));
preg_match_all("/([^\s\,]+)[\s|\,]+([^\n]+)/u",
$lex_data, $lex_parts);
$dictionary = array_combine($lex_parts[1], $lex_parts[2]);
}
}
$tokens = preg_split("/\s+/u", $text);
$result = [];
$tag_list = [];
$i = 0;
foreach ($tokens as $token)
{
//Tag the tokens as found in the Lexicon
$token = trim($token);
$current = ["token" => $token, "tag" => "UNKNOWN"];
$term = $current["token"];
if (!empty($dictionary[$token])) {
$tag_list = explode(" ", $dictionary[$token]);
$current['tag'] = $tag_list[0];
}
if (is_numeric($token)) {
$current["tag"] = "NN";
} else if (in_array($token, ["рд╣реИ", "рд╣реИрдВ"])) {
$current["tag"] = "VB";
}
if (empty($current["tag"])) {
$current["tag"] = "UNKNOWN";
}
$result[$i] = $current;
$i++;
}
$result = self::tagUnknownWords($result);
return $result;
}
/**
* This method tags the remaining words in a partially tagged text array.
*
* @param array $partially_tagged_text term array representing a text
* passage. Each element in array is in turnan associative array
* [token => token_value, tag => tag_value (may be empty)]
* @return array text passage array where all empty tags now have values
*/
public static function tagUnknownWords($partially_tagged_text)
{
$result = $partially_tagged_text;
$verbs = ["VBZ", "VBD", "VBN"];
$length = count($result);
$previous = $result[0];
for ($i = 1; $i < $length; $i++)
{
$current = $result[$i];
$current["token"] = trim($current["token"]);
$current["tag"] = trim($current["tag"]);
if ($current["tag"] == "UNKNOWN" || $previous["tag"] == "UNKNOWN") {
/**
* RULE 1: If the previous word tagged is a Adjective Pronoun
* Postposition then the current word is likely to be a noun
*/
if ($previous["tag"] == "JJ" ||
$previous["tag"] == "PRO_NN" ||
$previous["tag"] == "POST_POS") {
$current["tag"] = "NN";
$result[$i] = $current;
}
/**
* RULE 2: If the current word is a verb then the previous
* word is likely to be a noun
*/
if (in_array($current["tag"], $verbs)) {
$previous["tag"] = "NN";
$result[$i-1] = $previous;
}
/**
* PRONOUN IDENTIFICATION
* RULE 3: If the previous word is unknown and cuurent word
* is a noun then the previous word is most likely to be a
* pronoun
*/
if ($previous["tag"] == "UNKNOWN" &&
$current["tag"] == "NN") {
$previous["tag"] = "PRP";
$result[$i-1] = $previous;
}
/**
* VERB IDENTIFICATION
* RULE 4: If the current word is tagged as Auxilary verb and
* previous word is tagged as Unknown then most likely that
* the previous word is a verb
*/
if ($current["tag"] == "VAUX" &&
$previous["tag"] == "UNKNOWN") {
$previous["tag"] = "VB";
$result[$i-1] = $previous;
}
/**
* ADJECTIVE IDENTIFIATION
* RULE 5: if the currennt word ends with "рддрдо" or "рдЗрдХ" or "рд┐рдХ"
* or "рддрд░" then the word is an adjective
*/
if(mb_substr($current["token"], -2, 2) == "рдЗрдХ" ||
mb_substr($current["token"], -2, 2) == "рд┐рдХ" ||
mb_substr($current["token"], -2, 2) == "рддрд░" ||
mb_substr($current["token"], -2, 2) == "рддрдо") {
$current["tag"] = "JJ";
$result[$i] = $current;
}
if ($current["tag"] == "UNKNOWN") {
$current["tag"] = "NN";
$result[$i] = $current;
}
if ($previous["tag"] == "UNKNOWN"){
$previous["tag"] = "NN";
$result[$i-1] = $previous;
}
}
$previous = $current;
}
return $result;
}
/**
* This method is used to simplify the different tags of speech to a
* common form
*
* @param array $tagged_tokens which is an array of tokens assigned tags.
* @param bool $with_tokens whether to include the terms and the tags
* in the output string or just the part of speech tags
* @return string $tagged_phrase which is a string fo form token~pos
*/
public static function taggedPartOfSpeechTokensToString($tagged_tokens,
$with_tokens = true)
{
$tagged_phrase = "";
$with_tokens = $with_tokens;
$simplified_parts_of_speech = [
"NNS" => "NN", "NNP" => "NN", "NNPS" => "NN","WP" => "NN",
"VB" => "VB", "VBD" => "VB", "VBN" => "VB", "VBP" => "VB",
"VBZ" => "VB",
"JJ" => "AJ", "JJR" => "AJ", "JJS" => "AJ",
"RB" => "AV", "RBR" => "AV", "RBS" => "AV", "WRB" => "AV",
"inj" => "IN", "case" => "IN", "proNN" => "IN", "particle" => "IN",
"PREP" => "IN", "IN" => "IN", "PSP" => "IN",
"direct_DT" => "DT",
];
foreach ($tagged_tokens as $t) {
$tag = trim($t["tag"]);
$tag = (isset($simplified_parts_of_speech[$tag])) ?
$simplified_parts_of_speech[$tag] : $tag;
$token = ($with_tokens) ? $t["token"] . "~" : "";
$tagged_phrase .= $token . $tag . " ";
}
return $tagged_phrase;
}
/**
* Starting at the $cur_node in a $tagged_phrase parse tree for a Hindi
* sentence, create a phrase string for each of the next nodes
* which belong to part of speech group $type.
*
* @param array &$cur_node node within parse tree
* @param array $tagged_phrase parse tree for phrase
* @param string $type self::$noun_type, self::$verb_type, etc
* @return string phrase string involving only terms of that $type
*/
public static function parseTypeList(&$cur_node, $tagged_phrase, $type)
{
$tag_phrase_string = "";
$start_node = $cur_node;
$next_tag = (empty($tagged_phrase[$cur_node]['tag'])) ? "" :
trim($tagged_phrase[$cur_node]['tag']);
$allowed_conjuncts = [];
while ($next_tag && (in_array($next_tag, $type))) {
$tag_phrase_string .= " ". $tagged_phrase[$cur_node]['token'];
$cur_node++;
$next_tag = (empty($tagged_phrase[$cur_node]['tag'])) ? "" :
trim($tagged_phrase[$cur_node]['tag']);
}
return $tag_phrase_string;
}
/**
* Takes a part-of-speech tagged phrase and pre-tree with a
* parse-from position and builds a parse tree for a noun if possible
*
* @param array $tagged_phrase
* an array of pairs of the form ("token" => token_for_term,
* "tag" => part_of_speech_tag_for_term)
* @param array $tree that consists of ["curnode" =>
* current parse position in $tagged_phrase]
* @return array has fields
* "cur_node" index of how far we parsed $tagged_phrase
* "NN" a subarray with a token node for the noun string that was
* parsed
*/
public static function parseNoun($tagged_phrase, $tree)
{
//Combining multiple noun into one
$noun_string = self::parseTypeList($tree['cur_node'], $tagged_phrase,
self::$noun_type);
if (!empty($noun_string)) {
$tree["NN"] = $noun_string;
}
return $tree;
}
/**
* Takes a part-of-speech tagged phrase and pre-tree with a
* parse-from position and builds a parse tree for a verb if possible
*
* @param array $tagged_phrase
* an array of pairs of the form ("token" => token_for_term,
* "tag"=> part_of_speech_tag_for_term)
* @param array $tree that consists of ["curnode" =>
* current parse position in $tagged_phrase]
* @return array has fields
* "cur_node" index of how far we parsed $tagged_phrase
* "VB" a subarray with a token node for the verb string that was
* parsed
*/
public static function parseVerb($tagged_phrase, $tree)
{
$verb_string = self::parseTypeList($tree['cur_node'], $tagged_phrase,
self::$verb_type);
if (!empty($verb_string)) {
$tree["VB"] = $verb_string;
}
return $tree;
}
/**
* Takes a part-of-speech tagged phrase and pre-tree with a
* parse-from position and builds a parse tree for an adjective if possible
*
* @param array $tagged_phrase
* an array of pairs of the form ("token" => token_for_term,
* "tag"=> part_of_speech_tag_for_term)
* @param array $tree that consists of ["cur_node" =>
* current parse position in $tagged_phrase]
* @return array has fields
* "cur_node" index of how far we parsed $tagged_phrase
* "JJ" a subarray with a token node for the adjective that was
* parsed
*/
public static function parseAdjective($tagged_phrase, $tree)
{
$adjective_string = self::parseTypeList($tree['cur_node'],
$tagged_phrase, self::$adjective_type);
if (!empty($adjective_string)) {
$tree["JJ"] = $adjective_string;
}
return $tree;
}
/**
* Takes a part-of-speech tagged phrase and parse-tree with a
* parse-from position and builds a parse tree for a sequence of
* postpositional phrases if possible
*
* @param array $tagged_phrase
* an array of pairs of the form ("token" => token_for_term,
* "tag" => part_of_speech_tag_for_term)
* @param array $tree that consists of ["cur_node" =>
* current parse position in $tagged_phrase]
* @param int $index position in array to start from
* @return array has fields
* "cur_node" index of how far we parsed $tagged_phrase
*/
public static function parsePostpositionPhrase($tagged_phrase, $tree,
$index = 1)
{
$cur_node = $tree["cur_node"];
$tree_pp["cur_node"] = $tree["cur_node"];
if (isset ($tagged_phrase[$cur_node]["tag"]) &&
in_array($tagged_phrase[$cur_node]["tag"],
self::$postpositional_type)) {
$pp_string = self::parseTypeList($cur_node, $tagged_phrase,
self::$postpositional_type);
if (!empty($pp_string)) {
$tree_pp["IN_$index"] = $pp_string;
}
$adjective_string = self::parseTypeList($cur_node, $tagged_phrase,
self::$adjective_type);
if (!empty($adjective_string)) {
$tree_pp["JJ_$index"] = $adjective_string;
}
$nn_string = self::parseTypeList($cur_node, $tagged_phrase,
self::$noun_type);
if (!empty($nn_string)) {
$tree_pp["NN_$index"] = $nn_string;
}
$tree_pp["cur_node"] = $cur_node;
$tree_next = self::parsePostpositionPhrase($tagged_phrase,
$tree_pp, $index + 1);
$tree_pp = array_merge($tree_pp, $tree_next);
}
$tree["cur_node"] = $tree_pp["cur_node"];
unset($tree_pp["cur_node"]);
$tree["POST"] = $tree_pp;
return $tree;
}
/**
* Takes a part-of-speech tagged phrase and parse-tree with a
* parse-from position and builds a parse tree for a noun phrase if possible
*
* @param array $tagged_phrase
* an array of pairs of the form ("token" => token_for_term,
* "tag"=> part_of_speech_tag_for_term)
* @param array $tree that consists of ["curnode" =>
* current parse position in $tagged_phrase]
* @return array has fields
* "cur_node" index of how far we parsed $tagged_phrase
* "JJ" with value an Adjective subtree
* "NN" with value of a Noun Subtree
*/
public static function parseNounPhrase($tagged_phrase, $tree)
{
$cur_node = $tree["cur_node"];
$tree_jj = self::parseAdjective($tagged_phrase,
["cur_node" => $tree["cur_node"]]);
$tree_nn = self::parseNoun($tagged_phrase,
["cur_node" => $tree_jj["cur_node"]]);
if ($tree_nn["cur_node"] == $cur_node) {
$tree["NP"] = "";
return $tree;
}
$cur_node = $tree_nn["cur_node"];
unset($tree_jj["cur_node"], $tree_nn["cur_node"]);
return ["cur_node" => $cur_node, "NP" =>
["JJ" => $tree_jj, "NN" => $tree_nn] ];
}
/**
* Takes a part-of-speech tagged phrase and pre-tree with a
* parse-from position and builds a parse tree for a verb phrase if possible
*
* @param array $tagged_phrase
* an array of pairs of the form ("token" => token_for_term,
* "tag"=> part_of_speech_tag_for_term)
* @param array $tree that consists of ["curnode" =>
* current parse position in $tagged_phrase]
* @return array has fields
* "cur_node" index of how far we parsed $tagged_phrase
* "VP" a subarray with possible fields
* "VB" with value a verb subtree
*/
public static function parseVerbPhrase($tagged_phrase, $tree)
{
$cur_node = $tree["cur_node"];
$tree_vb = self::parseVerb($tagged_phrase, ["cur_node" => $cur_node]);
if ($tree_vb["cur_node"] == $cur_node) {
$tree["VP"] = [];
return $tree;
}
$cur_node = $tree_vb["cur_node"];
$postposition_string = self::parseTypeList($cur_node,
$tagged_phrase, self::$postpositional_type);
if (!empty($postposition_string)) {
$tree_vb["IN"] = $postposition_string;
}
$tree_np = self::parseNounPhrase($tagged_phrase,
["cur_node" => $cur_node]);;
if ($tree_np["cur_node"] != $cur_node) {
$cur_node = $tree_np["cur_node"];
unset($tree_vb["cur_node"], $tree_np["cur_node"]);
return ['cur_node' => $cur_node, 'VP' =>['VB' => $tree_vb,
'NP' => $tree_np["NP"]]];
}
unset($tree_vb["cur_node"]);
return ['cur_node' => $cur_node, 'VP' => ['VB' => $tree_vb]];
}
/**
* Given a part-of-speeech tagged phrase array generates a parse tree
* for the phrase using a recursive descent parser.
*
* @param array $tagged_phrase
* an array of pairs of the form ("token" => token_for_term,
* "tag"=> part_of_speech_tag_for_term)
* @param $tree this parameter is ignored but kept so as to match
* other methods such as @see parseNounPhrase in the recursive
* descent parser
* @return array used to represent a tree. The array has up to three fields
* $tree["cur_node"] index of how far we parsed our$tagged_phrase
* $tree["NP"] contains a subtree for a subject phrase
* $tree["POST"] contains a subtree for a object phrase
* $tree["VP"] contains a subtree for a predicate phrase
*/
public static function parseWholePhrase($tagged_phrase, $tree = [])
{
$tree_np = self::parseNounPhrase($tagged_phrase, ["cur_node" => 0]);
$tree_pp = self::parsePostpositionPhrase($tagged_phrase,
["cur_node" => $tree_np["cur_node"]] );
$tree_vp = self::parseVerbPhrase($tagged_phrase,
["cur_node" => $tree_pp["cur_node"]] );
$cur_node = $tree_vp["cur_node"];
unset($tree_np["cur_node"], $tree_pp["cur_node"], $tree_vp["cur_node"]);
return ["cur_node" => $cur_node, "NP" => $tree_np["NP"],
"POST" => $tree_pp["POST"], "VP" => $tree_vp["VP"]];
}
/**
* Scans a word list for phrases. For phrases found generate
* a list of question and answer pairs at two levels of granularity:
* CONCISE (using all terms in original phrase) and RAW (removing
* (adjectives, etc).
*
* @param array $word_and_phrase_list of statements
* @return array with two fields: QUESTION_LIST consisting of
* (SUBJECT, COMPLEMENT) where one of the components has been
* replaced with a question marker.
*/
public static function extractTripletsPhrases($word_and_phrase_list)
{
$triplets_list = [];
$question_list = [];
$question_answer_list = [];
$triplet_types = ["CONCISE", "RAW"];
foreach ($word_and_phrase_list as $word_and_phrase => $position_list) {
$sentence = $word_and_phrase;
$sentence = preg_replace("/\s+/u", " ", $word_and_phrase);
$sentence = trim($sentence);
$tagged_phrase = self::tagTokenizePartOfSpeech($sentence);
$parse_tree = self::parseWholePhrase($tagged_phrase,
["cur_node" => 0]);
$triplets = self::extractTripletsParseTree($parse_tree);
$extracted_triplets = self::rearrangeTripletsByType($triplets);
foreach ($triplet_types as $type) {
if (!empty($extracted_triplets[$type])) {
$triplets = $extracted_triplets[$type];
$questions = $triplets["QUESTION_LIST"];
foreach ($questions as $question) {
$question_list[$question] = $position_list;
}
$question_answer_list = array_merge($question_answer_list,
$triplets["QUESTION_ANSWER_LIST"]);
}
}
}
$out_triplets["QUESTION_LIST"] = $question_list;
$out_triplets["QUESTION_ANSWER_LIST"] = $question_answer_list;
return $out_triplets;
}
/**
* Takes phrase tree $tree and a part-of-speech $pos returns
* the deepest $pos only path in tree.
*
* @param array $tree phrase to extract type from
* @param string $pos the part of speech to extract
* @return string the label of deepest $pos only path in $tree
*/
public static function extractDeepestSpeechPartPhrase($tree, $pos)
{
$extract = "";
if (!empty($tree[$pos])) {
$extract = self::extractDeepestSpeechPartPhrase($tree[$pos], $pos);
}
if (!$extract && !empty($tree[$pos]) && !empty($tree[$pos][$pos])) {
$extract = $tree[$pos][$pos];
}
return $extract;
}
/**
* Takes a parse tree of a phrase or statement and returns an array
* with two fields CONCISE and RAW the former having the subject of
* the original phrase (as a string) the latter having the importart
* parts of the subject
*
* @param array $tree representation of a parse tree of a phrase
* @return array with two fields CONCISE and RAW as described above
*/
public static function extractSubjectParseTree($tree)
{
$subject = [];
if (!empty($tree["NP"])) {
$subject["CONCISE"] = self::extractDeepestSpeechPartPhrase(
$tree["NP"], "NN");
$raw_subject = "";
$it = new \RecursiveIteratorIterator(
new \RecursiveArrayIterator($tree["NP"]));
foreach ($it as $v) {
$raw_subject .= $v . " ";
}
$subject["RAW"]= $raw_subject;
} else {
$subject["CONCISE"] = "";
$subject["RAW"] = "";
}
return $subject;
}
/**
* Takes a parse tree of a phrase or statement and returns an array
* with two fields CONCISE and RAW the former having the predicate of
* the original phrase (as a string) the latter having the importart
* parts of the predicate
*
* @param array $tree representation of a parse tree of a phrase
* @return array with two fields CONCISE and RAW as described above
*/
public static function extractPredicateParseTree($tree)
{
$predicate = [];
if (!empty($tree["VP"])) {
$tree_vp = $tree["VP"];
$predicate["CONCISE"] = self::extractDeepestSpeechPartPhrase(
$tree_vp, "VB");
$raw_predicate = "";
if (!empty($tree_vp["VB"])) {
$tree_vb = $tree_vp["VB"];
$it = new \RecursiveIteratorIterator(
new \RecursiveArrayIterator($tree_vb));
foreach ($it as $v) {
$raw_predicate .= $v . " ";
}
$predicate["RAW"] = $raw_predicate;
}
} else {
$predicate["CONCISE"] = "";
$predicate["RAW"] = "";
}
return $predicate;
}
/**
* Takes a parse tree of a phrase or statement and returns an array
* with two fields CONCISE and RAW the former having the object of
* the original phrase (as a string) the latter having the importart
* parts of the object
*
* @param array $tree representation of a parse tree of a phrase
* @return array with two fields CONCISE and RAW as described above
*/
public static function extractObjectParseTree($tree)
{
$object = [];
if (!empty($tree["POST"])) {
$tree_pp = $tree["POST"];
if (!empty($tree_pp["NP"])) {
$np = $tree_pp["NP"];
$object["CONCISE"] = self::extractDeepestSpeechPartPhrase($np,
"NN");
} else {
$object["CONCISE"] = "";
}
$raw_object = "";
$it = new \RecursiveIteratorIterator(
new \RecursiveArrayIterator($tree_pp));
foreach ($it as $v) {
$raw_object .= $v . " ";
}
$object["RAW"] = $raw_object;
} else {
$object["CONCISE"] = "";
$object["RAW"] = "";
}
return $object;
}
/**
* Takes a parse tree of a phrase and computes subject, predicate, and
* object arrays. Each of these array consists of two components CONCISE and
* RAW, CONCISE corresponding to something more similar to the words in the
* original phrase and RAW to the case where extraneous words have been
* removed
*
* @param array $parse_tree a parse tree for a sentence
* @return array triplet array
*/
public static function extractTripletsParseTree($parse_tree)
{
$triplets = [];
$triplets["subject"] = self::extractSubjectParseTree($parse_tree);
$triplets["object"] = self::extractObjectParseTree($parse_tree);
$triplets["predicate"] = self::extractPredicateParseTree($parse_tree);
return $triplets;
}
/**
* Takes a triplets array with subject, predicate, object fields with
* CONCISE and RAW subfields and rearranges it to have two fields CONCISE
* and RAW with subject, predicate, object, and QUESTION_ANSWER_LIST
* subfields
*
* @param array $sub_pred_obj_triplets in format described above
* @return array $processed_triplets in format described above
*/
public static function rearrangeTripletsByType($sub_pred_obj_triplets)
{
$processed_triplet = [];
$processed_triplets["CONCISE"] =
self::extractTripletByType($sub_pred_obj_triplets, "CONCISE");
$processed_triplets["RAW"] =
self::extractTripletByType($sub_pred_obj_triplets, "RAW");
return $processed_triplets;
}
/**
* Takes a triplets array with subject, predicate, object fields with
* CONCISE, RAW subfields and produces triplets with $type subfield
* where $type is one of CONCISE and RAW and with subject, predicate,
* object and QUESTION_ANSWER_LIST subfields
*
* @param array $sub_pred_obj_triplets in format described above
* @param string $type either CONCISE or RAW
* @return array $triplets in format described above
*/
public static function extractTripletByType($sub_pred_obj_triplets, $type)
{
$triplets = [];
if (!empty($sub_pred_obj_triplets["subject"][$type])
&& !empty($sub_pred_obj_triplets["predicate"][$type])
&& !empty($sub_pred_obj_triplets["object"][$type])) {
$question_answer_triplets = [];
$sentence = [$sub_pred_obj_triplets["subject"][$type],
$sub_pred_obj_triplets["object"][$type],
$sub_pred_obj_triplets["predicate"][$type]];
$question_triplets = [];
for ($j = 0; $j < 2; $j++) {
for ($i = 0; $i < 3; $i++) {
$question = $sentence;
$question[$i] = self::$question_token;
$question_string = implode(" ", $question);
$question_string = trim($question_string);
$question_string = preg_replace("/\s+/u", " ",
$question_string);
$question_triplets[] = $question_string;
$question_answer_triplets[$question_string] =
preg_replace("/\s+/u", " ", $sentence[$i]);
}
}
$triplets["QUESTION_LIST"] = $question_triplets;
$triplets["QUESTION_ANSWER_LIST"] = $question_answer_triplets;
}
return $triplets;
}
/**
* Takes tagged question string starts with Who
* and returns question triplet from the question string
*
* @param string $tagged_question part-of-speech tagged question
* @param int $index current index in statement
* @return array parsed triplet
*/
public static function parseQuestion($tagged_question, $index)
{
$generated_questions = [];
$triplets = [];
$tree_np = self::parseNounPhrase($tagged_question,
["cur_node" => 0]);
$triplets["subject"] = self::extractSubjectParseTree($tree_np);
$tree_vp = self::parseVerbPhrase($tagged_question,
["cur_node" => $index + 1]);
$triplets["predicate"] = self::extractPredicateParseTree($tree_vp);
$triplet_types = ["CONCISE", "RAW"];
foreach ($triplet_types as $type) {
if (!empty($triplets["subject"][$type])
&& !empty($triplets["predicate"][$type])) {
$question = trim (trim($triplets["subject"][$type]) .
" " . self::$question_token .
" " . trim($triplets["predicate"][$type]));
$question = preg_replace("/\s+/u", " ", $question);
$generated_questions[$type][] = $question;
}
}
return $generated_questions;
}
/**
* Takes a phrase query entered by user and return true if it is question
* and false if not
*
* @param $phrase any statement
* @return bool returns true if statement is question
*/
public function isQuestion($phrase)
{
return preg_match(self::$question_pattern, $phrase);
}
/**
* Takes questions and returns the triplet from the question
*
* @param string $question question to parse
* @return array question triplet
*/
public static function questionParser($question)
{
$question = trim($question);
$question = preg_replace("/\s+/u", " ", $question);
$tagged_question = self::tagTokenizePartOfSpeech($question);
$index = -1;
foreach ($tagged_question as $i => $term_pos) {
if (preg_match(self::$question_pattern, $term_pos["token"])) {
$index = $i;
$term_pos["tag"] = "p_wh";
$tagged_question[$i] = $term_pos;
break;
}
}
return self::parseQuestion($tagged_question, $index);
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software