3 * New version of the difference engine
5 * Copyright © 2008 Guy Van den Broeck <guy@guyvdb.eu>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 * http://www.gnu.org/copyleft/gpl.html
23 * @ingroup DifferenceEngine
25 use MediaWiki\Diff\ComplexityException
;
28 * This diff implementation is mainly lifted from the LCS algorithm of the Eclipse project which
29 * in turn is based on Myers' "An O(ND) difference algorithm and its variations"
30 * (http://citeseer.ist.psu.edu/myers86ond.html) with range compression (see Wu et al.'s
31 * "An O(NP) Sequence Comparison Algorithm").
33 * This implementation supports an upper bound on the execution time.
35 * Some ideas (and a bit of code) are from analyze.c, from GNU
36 * diffutils-2.7, which can be found at:
37 * ftp://gnudist.gnu.org/pub/gnu/diffutils/diffutils-2.7.tar.gz
39 * Complexity: O((M + N)D) worst case time, O(M + N + D^2) expected time, O(M + N) space
41 * @author Guy Van den Broeck, Geoffrey T. Dairiki, Tim Starling
42 * @ingroup DifferenceEngine
55 protected $bailoutComplexity = 0;
58 private $maxDifferences;
59 private $lcsLengthCorrectedForHeuristic = false;
65 public $heuristicUsed;
67 function __construct( $tooLong = 2000000, $powLimit = 1.45 ) {
68 $this->tooLong
= $tooLong;
69 $this->powLimit
= $powLimit;
75 * @param string[] $from_lines
76 * @param string[] $to_lines
77 * @throws ComplexityException
81 public function diff( $from_lines, $to_lines ) {
82 // Diff and store locally
83 $this->diffInternal( $from_lines, $to_lines );
85 // Merge edits when possible
86 $this->shiftBoundaries( $from_lines, $this->removed
, $this->added
);
87 $this->shiftBoundaries( $to_lines, $this->added
, $this->removed
);
89 // Compute the edit operations.
90 $n_from = count( $from_lines );
91 $n_to = count( $to_lines );
95 while ( $xi < $n_from ||
$yi < $n_to ) {
96 assert( $yi < $n_to ||
$this->removed
[$xi] );
97 assert( $xi < $n_from ||
$this->added
[$yi] );
99 // Skip matching "snake".
101 while ( $xi < $n_from && $yi < $n_to
102 && !$this->removed
[$xi] && !$this->added
[$yi]
104 $copy[] = $from_lines[$xi++
];
108 $edits[] = new DiffOpCopy( $copy );
111 // Find deletes & adds.
113 while ( $xi < $n_from && $this->removed
[$xi] ) {
114 $delete[] = $from_lines[$xi++
];
118 while ( $yi < $n_to && $this->added
[$yi] ) {
119 $add[] = $to_lines[$yi++
];
122 if ( $delete && $add ) {
123 $edits[] = new DiffOpChange( $delete, $add );
124 } elseif ( $delete ) {
125 $edits[] = new DiffOpDelete( $delete );
127 $edits[] = new DiffOpAdd( $add );
135 * Sets the complexity (in comparison operations) that can't be exceeded
138 public function setBailoutComplexity( $value ) {
139 $this->bailoutComplexity
= $value;
143 * Adjust inserts/deletes of identical lines to join changes
144 * as much as possible.
146 * We do something when a run of changed lines include a
147 * line at one end and has an excluded, identical line at the other.
148 * We are free to choose which identical line is included.
149 * `compareseq' usually chooses the one at the beginning,
150 * but usually it is cleaner to consider the following identical line
151 * to be the "change".
153 * This is extracted verbatim from analyze.c (GNU diffutils-2.7).
155 * @param string[] $lines
156 * @param string[] $changed
157 * @param string[] $other_changed
159 private function shiftBoundaries( array $lines, array &$changed, array $other_changed ) {
163 assert( count( $lines ) == count( $changed ) );
164 $len = count( $lines );
165 $other_len = count( $other_changed );
169 * Scan forwards to find beginning of another run of changes.
170 * Also keep track of the corresponding point in the other file.
172 * Throughout this code, $i and $j are adjusted together so that
173 * the first $i elements of $changed and the first $j elements
174 * of $other_changed both contain the same number of zeros
176 * Furthermore, $j is always kept so that $j == $other_len or
177 * $other_changed[$j] == false.
179 while ( $j < $other_len && $other_changed[$j] ) {
183 while ( $i < $len && !$changed[$i] ) {
184 assert( $j < $other_len && !$other_changed[$j] );
187 while ( $j < $other_len && $other_changed[$j] ) {
198 // Find the end of this run of changes.
199 while ( ++
$i < $len && $changed[$i] ) {
205 * Record the length of this run of changes, so that
206 * we can later determine whether the run has grown.
208 $runlength = $i - $start;
211 * Move the changed region back, so long as the
212 * previous unchanged line matches the last changed one.
213 * This merges with previous changed regions.
215 while ( $start > 0 && $lines[$start - 1] == $lines[$i - 1] ) {
216 $changed[--$start] = 1;
217 $changed[--$i] = false;
218 while ( $start > 0 && $changed[$start - 1] ) {
222 while ( $other_changed[--$j] ) {
225 assert( $j >= 0 && !$other_changed[$j] );
229 * Set CORRESPONDING to the end of the changed run, at the last
230 * point where it corresponds to a changed run in the other file.
231 * CORRESPONDING == LEN means no such point has been found.
233 $corresponding = $j < $other_len ?
$i : $len;
236 * Move the changed region forward, so long as the
237 * first changed line matches the following unchanged one.
238 * This merges with following changed regions.
239 * Do this second, so that if there are no merges,
240 * the changed region is moved forward as far as possible.
242 while ( $i < $len && $lines[$start] == $lines[$i] ) {
243 $changed[$start++
] = false;
245 while ( $i < $len && $changed[$i] ) {
249 assert( $j < $other_len && !$other_changed[$j] );
251 if ( $j < $other_len && $other_changed[$j] ) {
253 while ( $j < $other_len && $other_changed[$j] ) {
258 } while ( $runlength != $i - $start );
261 * If possible, move the fully-merged run of changes
262 * back to a corresponding run in the other file.
264 while ( $corresponding < $i ) {
265 $changed[--$start] = 1;
268 while ( $other_changed[--$j] ) {
271 assert( $j >= 0 && !$other_changed[$j] );
277 * @param string[] $from
278 * @param string[] $to
279 * @throws ComplexityException
281 protected function diffInternal( array $from, array $to ) {
282 // remember initial lengths
286 $this->heuristicUsed
= false;
289 $removed = $m > 0 ?
array_fill( 0, $m, true ) : [];
290 $added = $n > 0 ?
array_fill( 0, $n, true ) : [];
292 // reduce the complexity for the next step (intentionally done twice)
293 // remove common tokens at the start
295 while ( $i < $m && $i < $n && $from[$i] === $to[$i] ) {
296 $removed[$i] = $added[$i] = false;
297 unset( $from[$i], $to[$i] );
301 // remove common tokens at the end
303 while ( $i +
$j <= $m && $i +
$j <= $n && $from[$m - $j] === $to[$n - $j] ) {
304 $removed[$m - $j] = $added[$n - $j] = false;
305 unset( $from[$m - $j], $to[$n - $j] );
309 $this->from
= $newFromIndex = $this->to
= $newToIndex = [];
311 // remove tokens not in both sequences
313 foreach ( $from as $key ) {
314 $shared[$key] = false;
317 foreach ( $to as $index => &$el ) {
318 if ( array_key_exists( $el, $shared ) ) {
322 $newToIndex[] = $index;
325 foreach ( $from as $index => &$el ) {
326 if ( $shared[$el] ) {
329 $newFromIndex[] = $index;
333 unset( $shared, $from, $to );
335 $this->m
= count( $this->from
);
336 $this->n
= count( $this->to
);
338 if ( $this->bailoutComplexity
> 0 && $this->m
* $this->n
> $this->bailoutComplexity
) {
339 throw new ComplexityException();
342 $this->removed
= $this->m
> 0 ?
array_fill( 0, $this->m
, true ) : [];
343 $this->added
= $this->n
> 0 ?
array_fill( 0, $this->n
, true ) : [];
345 if ( $this->m
== 0 ||
$this->n
== 0 ) {
348 $this->maxDifferences
= ceil( ( $this->m +
$this->n
) / 2.0 );
349 if ( $this->m
* $this->n
> $this->tooLong
) {
350 // limit complexity to D^POW_LIMIT for long sequences
351 $this->maxDifferences
= floor( pow( $this->maxDifferences
, $this->powLimit
- 1.0 ) );
352 wfDebug( "Limiting max number of differences to $this->maxDifferences\n" );
356 * The common prefixes and suffixes are always part of some LCS, include
357 * them now to reduce our search space
359 $max = min( $this->m
, $this->n
);
360 for ( $forwardBound = 0; $forwardBound < $max
361 && $this->from
[$forwardBound] === $this->to
[$forwardBound];
364 $this->removed
[$forwardBound] = $this->added
[$forwardBound] = false;
367 $backBoundL1 = $this->m
- 1;
368 $backBoundL2 = $this->n
- 1;
370 while ( $backBoundL1 >= $forwardBound && $backBoundL2 >= $forwardBound
371 && $this->from
[$backBoundL1] === $this->to
[$backBoundL2]
373 $this->removed
[$backBoundL1--] = $this->added
[$backBoundL2--] = false;
376 $temp = array_fill( 0, $this->m +
$this->n +
1, 0 );
377 $V = [ $temp, $temp ];
378 $snake = [ 0, 0, 0 ];
380 $this->length
= $forwardBound +
$this->m
- $backBoundL1 - 1
394 $this->length +
= $i +
$j - 1;
396 foreach ( $this->removed
as $key => &$removed_elem ) {
397 if ( !$removed_elem ) {
398 $removed[$newFromIndex[$key]] = false;
401 foreach ( $this->added
as $key => &$added_elem ) {
402 if ( !$added_elem ) {
403 $added[$newToIndex[$key]] = false;
406 $this->removed
= $removed;
407 $this->added
= $added;
410 function diff_range( $from_lines, $to_lines ) {
411 // Diff and store locally
412 $this->diff( $from_lines, $to_lines );
413 unset( $from_lines, $to_lines );
417 while ( $xi < $this->m ||
$yi < $this->n
) {
419 while ( $xi < $this->m
&& $yi < $this->n
420 && !$this->removed
[$xi]
421 && !$this->added
[$yi]
426 // Find deletes & adds.
428 while ( $xi < $this->m
&& $this->removed
[$xi] ) {
433 while ( $yi < $this->n
&& $this->added
[$yi] ) {
437 if ( $xi > $xstart ||
$yi > $ystart ) {
438 $ranges[] = new RangeDifference( $xstart, $xi, $ystart, $yi );
445 private function lcs_rec( $bottoml1, $topl1, $bottoml2, $topl2, &$V, &$snake ) {
446 // check that both sequences are non-empty
447 if ( $bottoml1 > $topl1 ||
$bottoml2 > $topl2 ) {
451 $d = $this->find_middle_snake( $bottoml1, $topl1, $bottoml2,
452 $topl2, $V, $snake );
454 // need to store these so we don't lose them when they're
455 // overwritten by the recursion
460 // the middle snake is part of the LCS, store it
461 for ( $i = 0; $i < $len; ++
$i ) {
462 $this->removed
[$startx +
$i] = $this->added
[$starty +
$i] = false;
467 +
$this->lcs_rec( $bottoml1, $startx - 1, $bottoml2,
468 $starty - 1, $V, $snake )
469 +
$this->lcs_rec( $startx +
$len, $topl1, $starty +
$len,
470 $topl2, $V, $snake );
471 } elseif ( $d == 1 ) {
473 * In this case the sequences differ by exactly 1 line. We have
474 * already saved all the lines after the difference in the for loop
475 * above, now we need to save all the lines before the difference.
477 $max = min( $startx - $bottoml1, $starty - $bottoml2 );
478 for ( $i = 0; $i < $max; ++
$i ) {
479 $this->removed
[$bottoml1 +
$i] =
480 $this->added
[$bottoml2 +
$i] = false;
489 private function find_middle_snake( $bottoml1, $topl1, $bottoml2, $topl2, &$V, &$snake ) {
490 $from = &$this->from
;
494 $snake0 = &$snake[0];
495 $snake1 = &$snake[1];
496 $snake2 = &$snake[2];
497 $bottoml1_min_1 = $bottoml1 - 1;
498 $bottoml2_min_1 = $bottoml2 - 1;
499 $N = $topl1 - $bottoml1_min_1;
500 $M = $topl2 - $bottoml2_min_1;
502 $maxabsx = $N +
$bottoml1;
503 $maxabsy = $M +
$bottoml2;
504 $limit = min( $this->maxDifferences
, ceil( ( $N +
$M ) / 2 ) );
506 // value_to_add_forward: a 0 or 1 that we add to the start
507 // offset to make it odd/even
508 if ( ( $M & 1 ) == 1 ) {
509 $value_to_add_forward = 1;
511 $value_to_add_forward = 0;
514 if ( ( $N & 1 ) == 1 ) {
515 $value_to_add_backward = 1;
517 $value_to_add_backward = 0;
520 $start_forward = -$M;
522 $start_backward = -$N;
525 $limit_min_1 = $limit - 1;
526 $limit_plus_1 = $limit +
1;
528 $V0[$limit_plus_1] = 0;
529 $V1[$limit_min_1] = $N;
530 $limit = min( $this->maxDifferences
, ceil( ( $N +
$M ) / 2 ) );
532 if ( ( $delta & 1 ) == 1 ) {
533 for ( $d = 0; $d <= $limit; ++
$d ) {
534 $start_diag = max( $value_to_add_forward +
$start_forward, -$d );
535 $end_diag = min( $end_forward, $d );
536 $value_to_add_forward = 1 - $value_to_add_forward;
538 // compute forward furthest reaching paths
539 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
540 if ( $k == -$d ||
( $k < $d
541 && $V0[$limit_min_1 +
$k] < $V0[$limit_plus_1 +
$k] )
543 $x = $V0[$limit_plus_1 +
$k];
545 $x = $V0[$limit_min_1 +
$k] +
1;
548 $absx = $snake0 = $x +
$bottoml1;
549 $absy = $snake1 = $x - $k +
$bottoml2;
551 while ( $absx < $maxabsx && $absy < $maxabsy && $from[$absx] === $to[$absy] ) {
555 $x = $absx - $bottoml1;
557 $snake2 = $absx - $snake0;
558 $V0[$limit +
$k] = $x;
559 if ( $k >= $delta - $d +
1 && $k <= $delta +
$d - 1
560 && $x >= $V1[$limit +
$k - $delta]
565 // check to see if we can cut down the diagonal range
566 if ( $x >= $N && $end_forward > $k - 1 ) {
567 $end_forward = $k - 1;
568 } elseif ( $absy - $bottoml2 >= $M ) {
569 $start_forward = $k +
1;
570 $value_to_add_forward = 0;
574 $start_diag = max( $value_to_add_backward +
$start_backward, -$d );
575 $end_diag = min( $end_backward, $d );
576 $value_to_add_backward = 1 - $value_to_add_backward;
578 // compute backward furthest reaching paths
579 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
581 ||
( $k != -$d && $V1[$limit_min_1 +
$k] < $V1[$limit_plus_1 +
$k] )
583 $x = $V1[$limit_min_1 +
$k];
585 $x = $V1[$limit_plus_1 +
$k] - 1;
588 $y = $x - $k - $delta;
591 while ( $x > 0 && $y > 0
592 && $from[$x +
$bottoml1_min_1] === $to[$y +
$bottoml2_min_1]
598 $V1[$limit +
$k] = $x;
600 // check to see if we can cut down our diagonal range
602 $start_backward = $k +
1;
603 $value_to_add_backward = 0;
604 } elseif ( $y <= 0 && $end_backward > $k - 1 ) {
605 $end_backward = $k - 1;
610 for ( $d = 0; $d <= $limit; ++
$d ) {
611 $start_diag = max( $value_to_add_forward +
$start_forward, -$d );
612 $end_diag = min( $end_forward, $d );
613 $value_to_add_forward = 1 - $value_to_add_forward;
615 // compute forward furthest reaching paths
616 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
618 ||
( $k < $d && $V0[$limit_min_1 +
$k] < $V0[$limit_plus_1 +
$k] )
620 $x = $V0[$limit_plus_1 +
$k];
622 $x = $V0[$limit_min_1 +
$k] +
1;
625 $absx = $snake0 = $x +
$bottoml1;
626 $absy = $snake1 = $x - $k +
$bottoml2;
628 while ( $absx < $maxabsx && $absy < $maxabsy && $from[$absx] === $to[$absy] ) {
632 $x = $absx - $bottoml1;
633 $snake2 = $absx - $snake0;
634 $V0[$limit +
$k] = $x;
636 // check to see if we can cut down the diagonal range
637 if ( $x >= $N && $end_forward > $k - 1 ) {
638 $end_forward = $k - 1;
639 } elseif ( $absy - $bottoml2 >= $M ) {
640 $start_forward = $k +
1;
641 $value_to_add_forward = 0;
645 $start_diag = max( $value_to_add_backward +
$start_backward, -$d );
646 $end_diag = min( $end_backward, $d );
647 $value_to_add_backward = 1 - $value_to_add_backward;
649 // compute backward furthest reaching paths
650 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
652 ||
( $k != -$d && $V1[$limit_min_1 +
$k] < $V1[$limit_plus_1 +
$k] )
654 $x = $V1[$limit_min_1 +
$k];
656 $x = $V1[$limit_plus_1 +
$k] - 1;
659 $y = $x - $k - $delta;
662 while ( $x > 0 && $y > 0
663 && $from[$x +
$bottoml1_min_1] === $to[$y +
$bottoml2_min_1]
669 $V1[$limit +
$k] = $x;
671 if ( $k >= -$delta - $d && $k <= $d - $delta
672 && $x <= $V0[$limit +
$k +
$delta]
674 $snake0 = $bottoml1 +
$x;
675 $snake1 = $bottoml2 +
$y;
680 // check to see if we can cut down our diagonal range
682 $start_backward = $k +
1;
683 $value_to_add_backward = 0;
684 } elseif ( $y <= 0 && $end_backward > $k - 1 ) {
685 $end_backward = $k - 1;
691 * computing the true LCS is too expensive, instead find the diagonal
692 * with the most progress and pretend a midle snake of length 0 occurs
696 $most_progress = self
::findMostProgress( $M, $N, $limit, $V );
698 $snake0 = $bottoml1 +
$most_progress[0];
699 $snake1 = $bottoml2 +
$most_progress[1];
701 wfDebug( "Computing the LCS is too expensive. Using a heuristic.\n" );
702 $this->heuristicUsed
= true;
705 * HACK: since we didn't really finish the LCS computation
706 * we don't really know the length of the SES. We don't do
707 * anything with the result anyway, unless it's <=1. We know
708 * for a fact SES > 1 so 5 is as good a number as any to
713 private static function findMostProgress( $M, $N, $limit, $V ) {
716 if ( ( $M & 1 ) == ( $limit & 1 ) ) {
717 $forward_start_diag = max( -$M, -$limit );
719 $forward_start_diag = max( 1 - $M, -$limit );
722 $forward_end_diag = min( $N, $limit );
724 if ( ( $N & 1 ) == ( $limit & 1 ) ) {
725 $backward_start_diag = max( -$N, -$limit );
727 $backward_start_diag = max( 1 - $N, -$limit );
730 $backward_end_diag = -min( $M, $limit );
734 $max_progress = array_fill( 0, ceil( max( $forward_end_diag - $forward_start_diag,
735 $backward_end_diag - $backward_start_diag ) / 2 ), $temp );
736 $num_progress = 0; // the 1st entry is current, it is initialized
739 // first search the forward diagonals
740 for ( $k = $forward_start_diag; $k <= $forward_end_diag; $k +
= 2 ) {
741 $x = $V[0][$limit +
$k];
743 if ( $x > $N ||
$y > $M ) {
748 if ( $progress > $max_progress[0][2] ) {
750 $max_progress[0][0] = $x;
751 $max_progress[0][1] = $y;
752 $max_progress[0][2] = $progress;
753 } elseif ( $progress == $max_progress[0][2] ) {
755 $max_progress[$num_progress][0] = $x;
756 $max_progress[$num_progress][1] = $y;
757 $max_progress[$num_progress][2] = $progress;
761 $max_progress_forward = true; // initially the maximum
762 // progress is in the forward
765 // now search the backward diagonals
766 for ( $k = $backward_start_diag; $k <= $backward_end_diag; $k +
= 2 ) {
767 $x = $V[1][$limit +
$k];
768 $y = $x - $k - $delta;
769 if ( $x < 0 ||
$y < 0 ) {
773 $progress = $N - $x +
$M - $y;
774 if ( $progress > $max_progress[0][2] ) {
776 $max_progress_forward = false;
777 $max_progress[0][0] = $x;
778 $max_progress[0][1] = $y;
779 $max_progress[0][2] = $progress;
780 } elseif ( $progress == $max_progress[0][2] && !$max_progress_forward ) {
782 $max_progress[$num_progress][0] = $x;
783 $max_progress[$num_progress][1] = $y;
784 $max_progress[$num_progress][2] = $progress;
788 // return the middle diagonal with maximal progress.
789 return $max_progress[(int)floor( $num_progress / 2 )];
795 public function getLcsLength() {
796 if ( $this->heuristicUsed
&& !$this->lcsLengthCorrectedForHeuristic
) {
797 $this->lcsLengthCorrectedForHeuristic
= true;
798 $this->length
= $this->m
- array_sum( $this->added
);
801 return $this->length
;
807 * Alternative representation of a set of changes, by the index
808 * ranges that are changed.
810 * @ingroup DifferenceEngine
812 class RangeDifference
{
832 function __construct( $leftstart, $leftend, $rightstart, $rightend ) {
833 $this->leftstart
= $leftstart;
834 $this->leftend
= $leftend;
835 $this->leftlength
= $leftend - $leftstart;
836 $this->rightstart
= $rightstart;
837 $this->rightend
= $rightend;
838 $this->rightlength
= $rightend - $rightstart;