diff --git a/src/executables/Fetcher.php b/src/executables/Fetcher.php index 2aded9716..894ed4cc2 100755 --- a/src/executables/Fetcher.php +++ b/src/executables/Fetcher.php @@ -959,7 +959,6 @@ class Fetcher implements CrawlConstants { $prefix = $this->fetcher_num . "-"; $dirs = glob(C\CRAWL_DIR.'/cache/*', GLOB_ONLYDIR); - $full_base_name = $prefix . self::archive_base_name; foreach ($dirs as $dir) { if (strlen( @@ -1403,7 +1402,7 @@ class Fetcher implements CrawlConstants (isset($this->found_sites[self::TO_CRAWL][$cs]) && count($this->found_sites[self::TO_CRAWL][$cs]) > 0) || isset($this->found_sites[self::INVERTED_INDEX][$cs])) { - $this->updateScheduler(); + $this->updateScheduler($cs); $at_least_once = false; } $cs = ($cs + 1) % $num_servers; @@ -1428,9 +1427,7 @@ class Fetcher implements CrawlConstants } else { $info[self::QUEUE_SERVERS] = $this->queue_servers; } - if (isset($info[self::CURRENT_SERVER])) { - $this->current_server = $info[self::CURRENT_SERVER]; - } else { + if (!isset($info[self::CURRENT_SERVER])) { $info[self::CURRENT_SERVER] = $this->current_server; } $update_fields = [ @@ -2620,10 +2617,11 @@ class Fetcher implements CrawlConstants * event that the server doesn't acknowledge it loops and tries again after * a delay until the post is successful. At this point, memory for this data * is freed. + * + * @param string $current_server queue server to update */ - public function updateScheduler() + public function updateScheduler($current_server) { - $current_server = $this->current_server; $queue_server = $this->queue_servers[$current_server]; L\crawlLog("Updating machine: " . $queue_server); $prefix = $this->fetcher_num . "-" . $this->channel . "-"; @@ -2700,14 +2698,14 @@ class Fetcher implements CrawlConstants unset($schedule_data); //handle mini inverted index if ($seen_cnt > 0 ) { - $this->buildMiniInvertedIndex(); + $this->buildMiniInvertedIndex($current_server); } if (isset($this->found_sites[self::INVERTED_INDEX][$current_server])) { L\crawlLog("Saving Mini Inverted Index..."); $this->found_sites[self::INVERTED_INDEX][$current_server] = $this->found_sites[self::INVERTED_INDEX][ $current_server]->save(true, true); - $compress_urls = $this->compressAndUnsetSeenUrls(); + $compress_urls = $this->compressAndUnsetSeenUrls($current_server); $len_urls = strlen($compress_urls); L\crawlLog("...Finish Compressing seen URLs."); $out_string = L\packInt($len_urls) . $compress_urls; @@ -2749,11 +2747,11 @@ class Fetcher implements CrawlConstants * links destined for the current queue server. Then unsets these * values from $this->found_sites * + * @param string $current_server server to compress and unset urls for * @return string of compressed urls */ - public function compressAndUnsetSeenUrls() + public function compressAndUnsetSeenUrls($current_server) { - $current_server = $this->current_server; $compress_urls = ""; if (!isset($this->found_sites[self::LINK_SEEN_URLS][ $current_server])) { @@ -2967,9 +2965,9 @@ class Fetcher implements CrawlConstants * burden on the queue server. The resulting mini index computed by * buildMiniInvertedIndex() is stored in * $this->found_sites[self::INVERTED_INDEX] - * + * @param string $current_server queue server to build inverted index for */ - public function buildMiniInvertedIndex() + public function buildMiniInvertedIndex($current_server) { $start_time = microtime(true); $keypad = "\x00\x00\x00\x00"; @@ -2981,10 +2979,9 @@ class Fetcher implements CrawlConstants for the fetcher we are not saving the index shards so name doesn't matter. */ - if (!isset($this->found_sites[self::INVERTED_INDEX][ - $this->current_server])) { - $this->found_sites[self::INVERTED_INDEX][$this->current_server] = - new IndexShard("fetcher_shard_{$this->current_server}"); + if (!isset($this->found_sites[self::INVERTED_INDEX][$current_server])) { + $this->found_sites[self::INVERTED_INDEX][$current_server] = + new IndexShard("fetcher_shard_{$current_server}"); } for ($i = 0; $i < $num_seen; $i++) { $interim_time = microtime(true); @@ -3099,7 +3096,7 @@ class Fetcher implements CrawlConstants $user_ranks = (empty($site[self::USER_RANKS])) ? [] : $site[self::USER_RANKS]; - $this->found_sites[self::INVERTED_INDEX][$this->current_server + $this->found_sites[self::INVERTED_INDEX][$current_server ]->addDocumentWords($doc_keys, self::NEEDS_OFFSET_FLAG, $word_lists, $meta_ids, true, $doc_rank, $description_scores, $user_ranks); diff --git a/src/locale/ar/configure.ini b/src/locale/ar/configure.ini index d6adedb92..7d96d8223 100755 --- a/src/locale/ar/configure.ini +++ b/src/locale/ar/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "بي إتش بي محرك البحث-يوب!: %s" rss_layout_description = "نتائج البحث ل: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/bn/configure.ini b/src/locale/bn/configure.ini index d4f4254c8..1c11c9a15 100755 --- a/src/locale/bn/configure.ini +++ b/src/locale/bn/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "পিএইচপি সার্চ ইঞ্জিন - Y rss_layout_description = "জন্য অনুসন্ধান ফলাফল: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/de/configure.ini b/src/locale/de/configure.ini index 1647e8b6e..83b4b2000 100755 --- a/src/locale/de/configure.ini +++ b/src/locale/de/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP Suchmaschine - Yioop! : %s" rss_layout_description = "Suchergebnisse für: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/en_US/configure.ini b/src/locale/en_US/configure.ini index 2f168c048..ff9a303c1 100644 --- a/src/locale/en_US/configure.ini +++ b/src/locale/en_US/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP Search Engine - Yioop! : %s" rss_layout_description = "Search results for: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/es/configure.ini b/src/locale/es/configure.ini index 7994547e5..57ddc964a 100755 --- a/src/locale/es/configure.ini +++ b/src/locale/es/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP Motor de Búsqueda - Yioop! : %s" rss_layout_description = "Resultados de la búsqueda por: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/fa/configure.ini b/src/locale/fa/configure.ini index 021cf2edf..847018d5e 100755 --- a/src/locale/fa/configure.ini +++ b/src/locale/fa/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "موتور جستجوی PHP - Yioop! : %s" rss_layout_description = "نتایج جستجو برای: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/fr_FR/configure.ini b/src/locale/fr_FR/configure.ini index 76c54bab2..7d37b8295 100755 --- a/src/locale/fr_FR/configure.ini +++ b/src/locale/fr_FR/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "Moteur de recherche PHP -Yioop! %s" rss_layout_description = "%s résultats" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/he/configure.ini b/src/locale/he/configure.ini index 4bf1c0807..158e1a361 100755 --- a/src/locale/he/configure.ini +++ b/src/locale/he/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP מנוע חיפוש - Yioop! : %s" rss_layout_description = "תוצאות חיפוש עבור: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/hi/configure.ini b/src/locale/hi/configure.ini index 3db494619..18ad62166 100755 --- a/src/locale/hi/configure.ini +++ b/src/locale/hi/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP खोज इंजन - Yioop! : %s" rss_layout_description = "खोज परिणाम के लिए: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/id/configure.ini b/src/locale/id/configure.ini index 73a883527..fa7f0785c 100755 --- a/src/locale/id/configure.ini +++ b/src/locale/id/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "Mesin Pencari PHP - Yioop! : %s" rss_layout_description = "Hasil pencarian untuk: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/it/configure.ini b/src/locale/it/configure.ini index 714af639d..4bca25007 100755 --- a/src/locale/it/configure.ini +++ b/src/locale/it/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "Yioop! Motore di Ricerca in PHP: %s" rss_layout_description = "Risultati di ricerca per: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/ja/configure.ini b/src/locale/ja/configure.ini index fba1b3c64..a1f08bde3 100755 --- a/src/locale/ja/configure.ini +++ b/src/locale/ja/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHPの検索エンジン-Yioop! :%s" rss_layout_description = "検索結果：%s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/kn/configure.ini b/src/locale/kn/configure.ini index 686855317..866b5a46e 100755 --- a/src/locale/kn/configure.ini +++ b/src/locale/kn/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "ಪಿಹೆಚಪಿ ಶೋಧನಾ ಯಂತ್ರ - ಯ rss_layout_description = "ಈ ಶೋಧನಾ ಫಲಿತಾಂಶಗಳು ನಿಮ್ಮ ಪ್ರಶ್ನೆ: %s ಯ ಉತ್ತರ" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/ko/configure.ini b/src/locale/ko/configure.ini index 73681afaa..431ab71f4 100755 --- a/src/locale/ko/configure.ini +++ b/src/locale/ko/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP 검색 엔진 - Yioop! : %s" rss_layout_description = "%s 에 대한 서치 결과:" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/nl/configure.ini b/src/locale/nl/configure.ini index d5d49eedc..a1da42d6c 100644 --- a/src/locale/nl/configure.ini +++ b/src/locale/nl/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP Search Engine - Yioop! :%S" rss_layout_description = "Zoek resultaten voor: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/pl/configure.ini b/src/locale/pl/configure.ini index 3c045a9fd..c0bd396c5 100755 --- a/src/locale/pl/configure.ini +++ b/src/locale/pl/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "Wyszukaj silniku PHP - Yioop! : %s" rss_layout_description = "Wyniki wyszukiwania dla: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/pt/configure.ini b/src/locale/pt/configure.ini index 3689a8d27..db11f93b4 100755 --- a/src/locale/pt/configure.ini +++ b/src/locale/pt/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP Motor de Busca - Yioop! : %s" rss_layout_description = "Resultados da pesquisa para: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/ru/configure.ini b/src/locale/ru/configure.ini index 8ea318114..dbc546366 100755 --- a/src/locale/ru/configure.ini +++ b/src/locale/ru/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "Поиск движке PHP - Yioop! : %s" rss_layout_description = "Результаты поиска для: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/te/configure.ini b/src/locale/te/configure.ini index a40acf366..3523892d8 100644 --- a/src/locale/te/configure.ini +++ b/src/locale/te/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP శోధన ఇంజిన్ - Yioop! : %s" rss_layout_description = "కోసం శోధన ఫలితాలు: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/th/configure.ini b/src/locale/th/configure.ini index 7e46d6352..41fe9394d 100755 --- a/src/locale/th/configure.ini +++ b/src/locale/th/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "ค้นหาเกี่ยวกับ phpquery เค rss_layout_description = "ผลการค้นหาสำหรับ:ทั้งหมด %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/tl/configure.ini b/src/locale/tl/configure.ini index e3080e41e..d2553690b 100644 --- a/src/locale/tl/configure.ini +++ b/src/locale/tl/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP Search Engine - Yioop! : %s" rss_layout_description = "Ang mga resulta ng paghahanap para sa: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/tr/configure.ini b/src/locale/tr/configure.ini index c720d16e1..08fa09e01 100755 --- a/src/locale/tr/configure.ini +++ b/src/locale/tr/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP Arama Motoru - Yioop! : %s" rss_layout_description = "%s için arama sonuçları: " ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/vi_VN/configure.ini b/src/locale/vi_VN/configure.ini index 4b7fe49c3..f0d42272e 100755 --- a/src/locale/vi_VN/configure.ini +++ b/src/locale/vi_VN/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "Công Cụ Tìm kiếm PHP - Yioop! : %s" rss_layout_description = "Kết quả tìm kiếm: %s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers diff --git a/src/locale/zh_CN/configure.ini b/src/locale/zh_CN/configure.ini index 5dd4122be..ce104bfbb 100755 --- a/src/locale/zh_CN/configure.ini +++ b/src/locale/zh_CN/configure.ini @@ -1778,7 +1778,7 @@ rss_layout_title = "PHP搜索引擎-Yioop! :%s" rss_layout_description = "搜索结果为：%s" ; ; View.php -view_locale_version = "8" +view_locale_version = "9" view_logo_alt_text = "Yioop" ; ; /src/views/helpers