2dcf7571942b33e94f8f6a0313a689f445aa3695
3 * Sync one file backend to another based on the journal of later.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
20 * @ingroup Maintenance
23 require_once( dirname( __FILE__
) . '/Maintenance.php' );
25 class SyncFileBackend
extends Maintenance
{
26 public function __construct() {
27 parent
::__construct();
28 $this->mDescription
= "Sync one file backend with another using the journal";
29 $this->addOption( 'src', 'Name of backend to sync from', true, true );
30 $this->addOption( 'dst', 'Name of destination backend to sync', true, true );
31 $this->addOption( 'start', 'Starting journal ID', false, true );
32 $this->addOption( 'end', 'Ending journal ID', false, true );
33 $this->addOption( 'posdir', 'Directory to read/record journal positions', false, true );
34 $this->addOption( 'verbose', 'Verbose mode', false, false, 'v' );
35 $this->setBatchSize( 50 );
38 public function execute() {
39 $src = FileBackendGroup
::singleton()->get( $this->getOption( 'src' ) );
40 $dst = FileBackendGroup
::singleton()->get( $this->getOption( 'dst' ) );
42 $posDir = $this->getOption( 'posdir' );
43 $posFile = $posDir ?
$posDir . '/' . wfWikiID() : false;
45 $start = $this->getOption( 'start', 0 );
46 if ( !$start && $posFile && is_dir( $posDir ) ) {
47 $start = is_file( $posFile )
48 ?
(int)trim( file_get_contents( $posFile ) )
50 ++
$start; // we already did this ID, start with the next one
51 $startFromPosFile = true;
53 $startFromPosFile = false;
55 $end = $this->getOption( 'end', INF
);
57 $this->output( "Synchronizing backend '{$dst->getName()}' to '{$src->getName()}'...\n" );
58 $this->output( "Starting journal position is $start.\n" );
59 if ( is_finite( $end ) ) {
60 $this->output( "Ending journal position is $end.\n" );
63 // Actually sync the dest backend with the reference backend
64 $lastOKPos = $this->syncBackends( $src, $dst, $start, $end );
66 // Update the sync position file
67 if ( $startFromPosFile && $lastOKPos >= $start ) { // successfully advanced
68 if ( file_put_contents( $posFile, $lastOKPos, LOCK_EX
) !== false ) {
69 $this->output( "Updated journal position file.\n" );
71 $this->output( "Could not update journal position file.\n" );
75 if ( $lastOKPos === false ) {
77 $this->output( "No journal entries found.\n" );
79 $this->output( "No new journal entries found.\n" );
82 $this->output( "Stopped synchronization at journal position $lastOKPos.\n" );
85 if ( $this->isQuiet() ) {
86 print $lastOKPos; // give a single machine-readable number
91 * Sync $dst backend to $src backend based on the $src logs given after $start.
92 * Returns the journal entry ID this advanced to and handled (inclusive).
94 * @param $src FileBackend
95 * @param $dst FileBackend
96 * @param $start integer Starting journal position
97 * @param $end integer Starting journal position
98 * @return integer|false Journal entry ID or false if there are none
100 protected function syncBackends( FileBackend
$src, FileBackend
$dst, $start, $end ) {
101 $lastOKPos = 0; // failed
102 $first = true; // first batch
104 if ( $start > $end ) { // sanity
105 $this->error( "Error: given starting ID greater than ending ID.", 1 );
109 $limit = min( $this->mBatchSize
, $end - $start +
1 ); // don't go pass ending ID
110 $this->output( "Doing id $start to " . ( $start +
$limit - 1 ) . "...\n" );
112 $entries = $src->getJournal()->getChangeEntries( $start, $limit, $next );
113 $start = $next; // start where we left off next time
114 if ( $first && !count( $entries ) ) {
115 return false; // nothing to do
120 $pathsInBatch = array(); // changed paths
121 foreach ( $entries as $entry ) {
122 if ( $entry['op'] !== 'null' ) { // null ops are just for reference
123 $pathsInBatch[$entry['path']] = 1; // remove duplicates
125 $lastPosInBatch = $entry['id'];
128 $status = $this->syncFileBatch( array_keys( $pathsInBatch ), $src, $dst );
129 if ( $status->isOK() ) {
130 $lastOKPos = max( $lastOKPos, $lastPosInBatch );
132 $this->error( print_r( $status->getErrorsArray(), true ) );
133 break; // no gaps; everything up to $lastPos must be OK
137 $this->output( "End of journal entries.\n" );
139 } while ( $start && $start <= $end );
145 * Sync particular files of backend $src to the corresponding $dst backend files
147 * @param $paths Array
148 * @param $src FileBackend
149 * @param $dst FileBackend
152 protected function syncFileBatch( array $paths, FileBackend
$src, FileBackend
$dst ) {
153 $status = Status
::newGood();
154 if ( !count( $paths ) ) {
155 return $status; // nothing to do
158 // Source: convert internal backend names (FileBackendMultiWrite) to the public one
159 $sPaths = $this->replaceNamePaths( $paths, $src );
160 // Destination: get corresponding path name
161 $dPaths = $this->replaceNamePaths( $paths, $dst );
163 // Lock the live backend paths from modification
164 $sLock = $src->getScopedFileLocks( $sPaths, LockManager
::LOCK_UW
, $status );
165 $eLock = $dst->getScopedFileLocks( $dPaths, LockManager
::LOCK_EX
, $status );
166 if ( !$status->isOK() ) {
172 foreach ( $sPaths as $i => $sPath ) {
173 $dPath = $dPaths[$i]; // destination
174 $sExists = $src->fileExists( array( 'src' => $sPath, 'latest' => 1 ) );
175 if ( $sExists === true ) { // exists in source
176 if ( $this->filesAreSame( $src, $dst, $sPath, $dPath ) ) {
177 continue; // avoid local copies for non-FS backends
179 // Note: getLocalReference() is fast for FS backends
180 $fsFile = $src->getLocalReference( array( 'src' => $sPath, 'latest' => 1 ) );
182 $this->error( "Unable to sync '$dPath': could not get local copy." );
183 $status->fatal( 'backend-fail-internal', $src->getName() );
186 $fsFiles[] = $fsFile; // keep TempFSFile objects alive as needed
187 // Note: prepare() is usually fast for key/value backends
188 $status->merge( $dst->prepare( array(
189 'dir' => dirname( $dPath ), 'bypassReadOnly' => 1 ) ) );
190 if ( !$status->isOK() ) {
193 $ops[] = array( 'op' => 'store',
194 'src' => $fsFile->getPath(), 'dst' => $dPath, 'overwrite' => 1 );
195 } elseif ( $sExists === false ) { // does not exist in source
196 $ops[] = array( 'op' => 'delete', 'src' => $dPath, 'ignoreMissingSource' => 1 );
198 $this->error( "Unable to sync '$dPath': could not stat file." );
199 $status->fatal( 'backend-fail-internal', $src->getName() );
204 $t_start = microtime( true );
205 $status->merge( $dst->doQuickOperations( $ops, array( 'bypassReadOnly' => 1 ) ) );
206 $ellapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 );
207 if ( $status->isOK() && $this->getOption( 'verbose' ) ) {
208 $this->output( "Synchronized these file(s) [{$ellapsed_ms}ms]:\n" .
209 implode( "\n", $dPaths ) . "\n" );
216 * Substitute the backend name of storage paths with that of a given one
218 * @param $paths Array|string List of paths or single string path
219 * @return Array|string
221 protected function replaceNamePaths( $paths, FileBackend
$backend ) {
223 '!^mwstore://([^/]+)!',
224 StringUtils
::escapeRegexReplacement( "mwstore://" . $backend->getName() ),
225 $paths // string or array
229 protected function filesAreSame( FileBackend
$src, FileBackend
$dst, $sPath, $dPath ) {
231 ( $src->getFileSize( array( 'src' => $sPath ) )
232 === $dst->getFileSize( array( 'src' => $dPath ) ) // short-circuit
233 ) && ( $src->getFileSha1Base36( array( 'src' => $sPath ) )
234 === $dst->getFileSha1Base36( array( 'src' => $dPath ) )
240 $maintClass = "SyncFileBackend";
241 require_once( RUN_MAINTENANCE_IF_MAIN
);