From 58a67086c65399f239b6cfd2a8f9c29eeafd7342 Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Tue, 22 Jan 2013 13:54:01 -0800 Subject: [PATCH] Improved concurrency of copyFileBackend.php with pipelined downloads. Change-Id: I0bde212cbba527d89b0c43f5174a77788cd07b71 --- maintenance/copyFileBackend.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/maintenance/copyFileBackend.php b/maintenance/copyFileBackend.php index aebdee175f..4e3c7faf77 100644 --- a/maintenance/copyFileBackend.php +++ b/maintenance/copyFileBackend.php @@ -134,6 +134,17 @@ class CopyFileBackend extends Maintenance { $ops = array(); $fsFiles = array(); $copiedRel = array(); // for output message + + // Download the batch of source files into backend cache... + if ( $this->hasOption( 'missingonly' ) ) { + $srcPaths = array(); + foreach ( $srcPathsRel as $srcPathRel ) { + $srcPaths[] = $src->getRootStoragePath() . "/$backendRel/$srcPathRel"; + } + $fsFiles = $src->getLocalReferenceMulti( array( 'srcs' => $srcPaths, 'latest' => 1 ) ); + } + + // Determine what files need to be copied over... foreach ( $srcPathsRel as $srcPathRel ) { $srcPath = $src->getRootStoragePath() . "/$backendRel/$srcPathRel"; $dstPath = $dst->getRootStoragePath() . "/$backendRel/$srcPathRel"; @@ -144,8 +155,9 @@ class CopyFileBackend extends Maintenance { $this->output( "Already have $srcPathRel.\n" ); continue; // assume already copied... } - // Note: getLocalReference() is fast for FS backends - $fsFile = $src->getLocalReference( array( 'src' => $srcPath, 'latest' => 1 ) ); + $fsFile = array_key_exists( $srcPath, $fsFiles ) + ? $fsFiles[$srcPath] + : $src->getLocalReference( array( 'src' => $srcPath, 'latest' => 1 ) ); if ( !$fsFile ) { $this->error( "Could not get local copy of $srcPath.", 1 ); // die } elseif ( !$fsFile->exists() ) { @@ -167,6 +179,7 @@ class CopyFileBackend extends Maintenance { $copiedRel[] = $srcPathRel; } + // Copy in the batch of source files... $t_start = microtime( true ); $status = $dst->doQuickOperations( $ops, array( 'bypassReadOnly' => 1 ) ); if ( !$status->isOK() ) { -- 2.20.1