Implement an iterator for build log chunks

Summary: Ref T5822. This will make it easier to compress and archive chunks without needing to hold them in memory.

Test Plan: Ran a build, looked at some logs.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T5822

Differential Revision: https://secure.phabricator.com/D15378

Author: epriestley

src/__phutil_library_map__.php

@@ -1045,6 +1045,7 @@
     'HarbormasterBuildLintMessage' => 'applications/harbormaster/storage/build/HarbormasterBuildLintMessage.php',
     'HarbormasterBuildLog' => 'applications/harbormaster/storage/build/HarbormasterBuildLog.php',
     'HarbormasterBuildLogChunk' => 'applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php',
+    'HarbormasterBuildLogChunkIterator' => 'applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php',
     'HarbormasterBuildLogPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php',
     'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php',
     'HarbormasterBuildMessage' => 'applications/harbormaster/storage/HarbormasterBuildMessage.php',
@@ -5193,6 +5194,7 @@
       'PhabricatorPolicyInterface',
     ),
     'HarbormasterBuildLogChunk' => 'HarbormasterDAO',
+    'HarbormasterBuildLogChunkIterator' => 'PhutilBufferedIterator',
     'HarbormasterBuildLogPHIDType' => 'PhabricatorPHIDType',
     'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'HarbormasterBuildMessage' => array(

src/applications/harbormaster/storage/build/HarbormasterBuildLog.php

@@ -16,11 +16,6 @@ final class HarbormasterBuildLog
 
   const CHUNK_BYTE_LIMIT = 102400;
 
-  /**
-   * The log is encoded as plain text.
-   */
-  const ENCODING_TEXT = 'text';
-
   public function __construct() {
     $this->rope = new PhutilRope();
   }
@@ -129,6 +124,8 @@ private function flush() {
 
     $chunk_table = id(new HarbormasterBuildLogChunk())->getTableName();
     $chunk_limit = self::CHUNK_BYTE_LIMIT;
+    $encoding_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
+
     $rope = $this->rope;
 
     while (true) {
@@ -147,7 +144,7 @@ private function flush() {
 
       $can_append =
         ($tail) &&
-        ($tail['encoding'] == self::ENCODING_TEXT) &&
+        ($tail['encoding'] == $encoding_text) &&
         ($tail['size'] < $chunk_limit);
       if ($can_append) {
         $append_id = $tail['id'];
@@ -176,7 +173,7 @@ private function flush() {
             VALUES (%d, %s, %d, %B)',
           $chunk_table,
           $this->getID(),
-          self::ENCODING_TEXT,
+          $encoding_text,
           $data_size,
           $append_data);
       }
@@ -185,29 +182,21 @@ private function flush() {
     }
   }
 
+  public function newChunkIterator() {
+    return new HarbormasterBuildLogChunkIterator($this);
+  }
+
   public function getLogText() {
-    // TODO: This won't cope very well if we're pulling like a 700MB
-    // log file out of the DB. We should probably implement some sort
-    // of optional limit parameter so that when we're rendering out only
-    // 25 lines in the UI, we don't wastefully read in the whole log.
-
-    // We have to read our content out of the database and stitch all of
-    // the log data back together.
-    $conn = $this->establishConnection('r');
-    $result = queryfx_all(
-      $conn,
-      'SELECT chunk '.
-      'FROM %T '.
-      'WHERE logID = %d '.
-      'ORDER BY id ASC',
-      id(new HarbormasterBuildLogChunk())->getTableName(),
-      $this->getID());
-
-    $content = '';
-    foreach ($result as $row) {
-      $content .= $row['chunk'];
+    // TODO: Remove this method since it won't scale for big logs.
+
+    $all_chunks = $this->newChunkIterator();
+
+    $full_text = array();
+    foreach ($all_chunks as $chunk) {
+      $full_text[] = $chunk->getChunkDisplayText();
     }
-    return $content;
+
+    return implode('', $full_text);
   }
 
 

src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php

@@ -8,6 +8,12 @@ final class HarbormasterBuildLogChunk
   protected $size;
   protected $chunk;
 
+
+  /**
+   * The log is encoded as plain text.
+   */
+  const CHUNK_ENCODING_TEXT = 'text';
+
   protected function getConfiguration() {
     return array(
       self::CONFIG_TIMESTAMPS => false,
@@ -29,4 +35,21 @@ protected function getConfiguration() {
     ) + parent::getConfiguration();
   }
 
+  public function getChunkDisplayText() {
+    $data = $this->getChunk();
+    $encoding = $this->getEncoding();
+
+    switch ($encoding) {
+      case self::CHUNK_ENCODING_TEXT:
+        // Do nothing, data is already plaintext.
+        break;
+      default:
+        throw new Exception(
+          pht('Unknown log chunk encoding ("%s")!', $encoding));
+    }
+
+    return $data;
+  }
+
+
 }

src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php

@@ -0,0 +1,35 @@
+<?php
+
+final class HarbormasterBuildLogChunkIterator
+  extends PhutilBufferedIterator {
+
+  private $log;
+  private $cursor;
+
+  public function __construct(HarbormasterBuildLog $log) {
+    $this->log = $log;
+  }
+
+  protected function didRewind() {
+    $this->cursor = 0;
+  }
+
+  public function key() {
+    return $this->current()->getID();
+  }
+
+  protected function loadPage() {
+    $results = id(new HarbormasterBuildLogChunk())->loadAllWhere(
+      'logID = %d AND id > %d ORDER BY id ASC LIMIT %d',
+      $this->log->getID(),
+      $this->cursor,
+      $this->getPageSize());
+
+    if ($results) {
+      $this->cursor = last($results)->getID();
+    }
+
+    return $results;
+  }
+
+}