viewgit/inc/functions.php:22 Function utf8_encode() is deprecated [8192]

Reduce memory requirements of save without dictionary, a=chris

Chris Pollett [2018-06-19 01:Jun:th]
Reduce memory requirements of save without dictionary, a=chris
Filename
src/library/IndexShard.php
diff --git a/src/library/IndexShard.php b/src/library/IndexShard.php
index a5956eba9..0bd9c4c24 100644
--- a/src/library/IndexShard.php
+++ b/src/library/IndexShard.php
@@ -1323,19 +1323,32 @@ class IndexShard extends PersistentStructure implements
             crawlLog("..without dictionary version of shard header written");
         }
         $header = "";
-        $word_docs = $this->getWordDocsSubstring();
-        fwrite($fh, $word_docs);
+        $remaining = $this->word_docs_len;
+        $offset = 0;
+        $buffer_size = 16 * self::SHARD_BLOCK_SIZE;
+        while ($remaining > 0) {
+            $len = min($remaining, $buffer_size);
+            $data = $this->getWordDocsSubstring($offset, $len);
+            fwrite($fh, $data);
+            $offset += $len;
+            $remaining -= $len;
+        }
         if($with_logging) {
             crawlLog("..without dictionary version of shard word docs written");
         }
-        $word_docs = "";
-        $doc_infos = $this->getDocInfoSubstring();
-        fwrite($fh, $doc_infos);
+        $remaining = $this->docids_len;
+        $offset = 0;
+        while ($remaining > 0) {
+            $len = min($remaining, $buffer_size);
+            $data = $this->getDocInfoSubstring($offset, $len);
+            fwrite($fh, $data);
+            $offset += $len;
+            $remaining -= $len;
+        }
         if($with_logging) {
             crawlLog("..without dictionary version of shard doc infos written");
         }
         fclose($fh);
-        $doc_infos = "";
         if (file_exists($this->filename . "-tmp")) {
             unlink($this->filename);
             rename($this->filename . "-tmp", $this->filename);
ViewGit