maintenance/postgres/compare_schemas.pl

   1 #!/usr/bin/perl
   2
   3 ## Rough check that the base and postgres "tables.sql" are in sync
   4 ## Should be run from maintenance/postgres
   5
   6 use strict;
   7 use warnings;
   8 use Data::Dumper;
   9
  10 my @old = ('../tables.sql');
  11 my $new = 'tables.sql';
  12 my @xfile;
  13
  14 ## Read in exceptions and other metadata
  15 my %ok;
  16 while (<DATA>) {
  17         next unless /^(\w+)\s*:\s*([^#]+)/;
  18         my ($name,$val) = ($1,$2);
  19         chomp $val;
  20         if ($name eq 'RENAME') {
  21                 die "Invalid rename\n" unless $val =~ /(\w+)\s+(\w+)/;
  22                 $ok{OLD}{$1} = $2;
  23                 $ok{NEW}{$2} = $1;
  24                 next;
  25         }
  26         if ($name eq 'XFILE') {
  27                 push @xfile, $val;
  28                 next;
  29         }
  30         for (split /\s+/ => $val) {
  31                 $ok{$name}{$_} = 0;
  32         }
  33 }
  34
  35 my $datatype = join '|' => qw(
  36 bool
  37 tinyint int bigint real float
  38 tinytext mediumtext text char varchar varbinary binary
  39 timestamp datetime
  40 tinyblob mediumblob blob
  41 );
  42 $datatype .= q{|ENUM\([\"\w, ]+\)};
  43 $datatype = qr{($datatype)};
  44
  45 my $typeval = qr{(\(\d+\))?};
  46
  47 my $typeval2 = qr{ unsigned| binary| NOT NULL| NULL| auto_increment| default ['\-\d\w"]+| REFERENCES .+CASCADE};
  48
  49 my $indextype = join '|' => qw(INDEX KEY FULLTEXT), 'PRIMARY KEY', 'UNIQUE INDEX', 'UNIQUE KEY';
  50 $indextype = qr{$indextype};
  51
  52 my $engine = qr{TYPE|ENGINE};
  53
  54 my $tabletype = qr{InnoDB|MyISAM|HEAP|HEAP MAX_ROWS=\d+|InnoDB MAX_ROWS=\d+ AVG_ROW_LENGTH=\d+};
  55
  56 my $charset = qr{utf8|binary};
  57
  58 open my $newfh, '<', $new or die qq{Could not open $new: $!\n};
  59
  60
  61 my ($table,%old);
  62
  63 ## Read in the xfiles
  64 my %xinfo;
  65 for my $xfile (@xfile) {
  66         print "Loading $xfile\n";
  67         my $info = &parse_sql($xfile);
  68         for (keys %$info) {
  69                 $xinfo{$_} = $info->{$_};
  70         }
  71 }
  72
  73 for my $oldfile (@old) {
  74         print "Loading $oldfile\n";
  75         my $info = &parse_sql($oldfile);
  76         for (keys %xinfo) {
  77                 $info->{$_} = $xinfo{$_};
  78         }
  79         $old{$oldfile} = $info;
  80 }
  81
  82 sub parse_sql {
  83
  84         my $oldfile = shift;
  85
  86         open my $oldfh, '<', $oldfile or die qq{Could not open $oldfile: $!\n};
  87
  88         my %info;
  89         while (<$oldfh>) {
  90                 next if /^\s*\-\-/ or /^\s+$/;
  91                 s/\s*\-\- [\w ]+$//;
  92                 chomp;
  93
  94                 if (/CREATE\s*TABLE/i) {
  95                         m{^CREATE TABLE /\*\$wgDBprefix\*/(\w+) \($}
  96                                 or die qq{Invalid CREATE TABLE at line $. of $oldfile\n};
  97                         $table = $1;
  98                         $info{$table}{name}=$table;
  99                 }
 100                 elsif (m#^\) /\*\$wgDBTableOptions\*/#) {
 101                         $info{$table}{engine} = 'TYPE';
 102                         $info{$table}{type} = 'variable';
 103                 }
 104                 elsif (/^\) ($engine)=($tabletype);$/) {
 105                         $info{$table}{engine}=$1;
 106                         $info{$table}{type}=$2;
 107                 }
 108                 elsif (/^\) ($engine)=($tabletype), DEFAULT CHARSET=($charset);$/) {
 109                         $info{$table}{engine}=$1;
 110                         $info{$table}{type}=$2;
 111                         $info{$table}{charset}=$3;
 112                 }
 113                 elsif (/^  (\w+) $datatype$typeval$typeval2{0,3},?$/) {
 114                         $info{$table}{column}{$1} = $2;
 115                         my $extra = $3 || '';
 116                         $info{$table}{columnfull}{$1} = "$2$extra";
 117                 }
 118                 elsif (/^  ($indextype)(?: (\w+))? \(([\w, \(\)]+)\),?$/) {
 119                         $info{$table}{lc $1.'_name'} = $2 ? $2 : '';
 120                         $info{$table}{lc $1.'pk_target'} = $3;
 121                 }
 122                 else {
 123                         die "Cannot parse line $. of $oldfile:\n$_\n";
 124                 }
 125
 126         }
 127         close $oldfh;
 128
 129         return \%info;
 130
 131 } ## end of parse_sql
 132
 133 ## Read in the parser test information
 134 my $parsefile = '../parserTests.inc';
 135 open my $pfh, '<', $parsefile or die qq{Could not open "$parsefile": $!\n};
 136 my $stat = 0;
 137 my %ptable;
 138 while (<$pfh>) {
 139         if (!$stat) {
 140                 if (/function listTables/) {
 141                         $stat = 1;
 142                 }
 143                 next;
 144         }
 145         $ptable{$1}=2 while /'(\w+)'/g;
 146         last if /\);/;
 147 }
 148 close $pfh;
 149
 150 my $OK_NOT_IN_PTABLE = '
 151 filearchive
 152 logging
 153 profiling
 154 querycache_info
 155 searchindex
 156 trackbacks
 157 transcache
 158 user_newtalk
 159 page_props
 160 ';
 161
 162 ## Make sure all tables in main tables.sql are accounted for int the parsertest.
 163 for my $table (sort keys %{$old{'../tables.sql'}}) {
 164         $ptable{$table}++;
 165         next if $ptable{$table} > 2;
 166         next if $OK_NOT_IN_PTABLE =~ /\b$table\b/;
 167         print qq{Table "$table" is in the schema, but not used inside of parserTest.inc\n};
 168 }
 169 ## Any that are used in ptables but no longer exist in the schema?
 170 for my $table (sort grep { $ptable{$_} == 2 } keys %ptable) {
 171         print qq{Table "$table" ($ptable{$table}) used in parserTest.inc, but not found in schema\n};
 172 }
 173
 174 for my $oldfile (@old) {
 175
 176 ## Begin non-standard indent
 177
 178 ## MySQL sanity checks
 179 for my $table (sort keys %{$old{$oldfile}}) {
 180         my $t = $old{$oldfile}{$table};
 181         if (($oldfile =~ /5/ and $t->{engine} ne 'ENGINE')
 182                 or
 183                 ($oldfile !~ /5/ and $t->{engine} ne 'TYPE')) {
 184                 die "Invalid engine for $oldfile: $t->{engine}\n" unless $t->{name} eq 'profiling';
 185         }
 186         my $charset = $t->{charset} || '';
 187         if ($oldfile !~ /binary/ and $charset eq 'binary') {
 188                 die "Invalid charset for $oldfile: $charset\n";
 189         }
 190 }
 191
 192 my $dtype = join '|' => qw(
 193 SMALLINT INTEGER BIGINT NUMERIC SERIAL
 194 TEXT CHAR VARCHAR
 195 BYTEA
 196 TIMESTAMPTZ
 197 CIDR
 198 );
 199 $dtype = qr{($dtype)};
 200 my %new;
 201 my ($infunction,$inview,$inrule,$lastcomma) = (0,0,0,0);
 202 seek $newfh, 0, 0;
 203 while (<$newfh>) {
 204         next if /^\s*\-\-/ or /^\s*$/;
 205         s/\s*\-\- [\w ']+$//;
 206         next if /^BEGIN;/ or /^SET / or /^COMMIT;/;
 207         next if /^CREATE SEQUENCE/;
 208         next if /^CREATE(?: UNIQUE)? INDEX/;
 209         next if /^CREATE FUNCTION/;
 210         next if /^CREATE TRIGGER/ or /^  FOR EACH ROW/;
 211         next if /^INSERT INTO/ or /^  VALUES \(/;
 212         next if /^ALTER TABLE/;
 213         chomp;
 214
 215         if (/^\$mw\$;?$/) {
 216                 $infunction = $infunction ? 0 : 1;
 217                 next;
 218         }
 219         next if $infunction;
 220
 221         next if /^CREATE VIEW/ and $inview = 1;
 222         if ($inview) {
 223                 /;$/ and $inview = 0;
 224                 next;
 225         }
 226
 227         next if /^CREATE RULE/ and $inrule = 1;
 228         if ($inrule) {
 229                 /;$/ and $inrule = 0;
 230                 next;
 231         }
 232
 233         if (/^CREATE TABLE "?(\w+)"? \($/) {
 234                 $table = $1;
 235                 $new{$table}{name}=$table;
 236                 $lastcomma = 1;
 237         }
 238         elsif (/^\);$/) {
 239                 if ($lastcomma) {
 240                         warn "Stray comma before line $.\n";
 241                 }
 242         }
 243         elsif (/^  (\w+) +$dtype.*?(,?)(?: --.*)?$/) {
 244                 $new{$table}{column}{$1} = $2;
 245                 if (!$lastcomma) {
 246                         print "Missing comma before line $. of $new\n";
 247                 }
 248                 $lastcomma = $3 ? 1 : 0;
 249         }
 250         else {
 251                 die "Cannot parse line $. of $new:\n$_\n";
 252         }
 253 }
 254
 255 ## Which column types are okay to map from mysql to postgres?
 256 my $COLMAP = q{
 257 ## INTS:
 258 tinyint SMALLINT
 259 int INTEGER SERIAL
 260 bigint BIGINT
 261 real NUMERIC
 262 float NUMERIC
 263
 264 ## TEXT:
 265 varchar(32) TEXT
 266 varchar(70) TEXT
 267 varchar(255) TEXT
 268 varchar TEXT
 269 text TEXT
 270 tinytext TEXT
 271 ENUM TEXT
 272
 273 ## TIMESTAMPS:
 274 varbinary(14) TIMESTAMPTZ
 275 binary(14) TIMESTAMPTZ
 276 datetime TIMESTAMPTZ
 277 timestamp TIMESTAMPTZ
 278
 279 ## BYTEA:
 280 mediumblob BYTEA
 281
 282 ## OTHER:
 283 bool SMALLINT # Sigh
 284
 285 };
 286 ## Allow specific exceptions to the above
 287 my $COLMAPOK = q{
 288 ## User inputted text strings:
 289 ar_comment      tinyblob       TEXT
 290 fa_description  tinyblob       TEXT
 291 img_description tinyblob       TEXT
 292 ipb_reason      tinyblob       TEXT
 293 log_action      varbinary(10)  TEXT
 294 oi_description  tinyblob       TEXT
 295 rev_comment     tinyblob       TEXT
 296 rc_log_action   varbinary(255) TEXT
 297 rc_log_type     varbinary(255) TEXT
 298
 299 ## Simple text-only strings:
 300 ar_flags          tinyblob       TEXT
 301 fa_minor_mime     varbinary(32)  TEXT
 302 fa_storage_group  varbinary(16)  TEXT # Just 'deleted' for now, should stay plain text
 303 fa_storage_key    varbinary(64)  TEXT # sha1 plus text extension
 304 ipb_address       tinyblob       TEXT # IP address or username
 305 ipb_range_end     tinyblob       TEXT # hexadecimal
 306 ipb_range_start   tinyblob       TEXT # hexadecimal
 307 img_minor_mime    varbinary(32)  TEXT
 308 img_sha1          varbinary(32)  TEXT
 309 job_cmd           varbinary(60)  TEXT # Should we limit to 60 as well?
 310 keyname           varbinary(255) TEXT # No tablename prefix (objectcache)
 311 ll_lang           varbinary(20)  TEXT # Language code
 312 log_params        blob           TEXT # LF separated list of args
 313 log_type          varbinary(10)  TEXT
 314 oi_minor_mime     varbinary(32)  TEXT
 315 oi_sha1           varbinary(32)  TEXT
 316 old_flags         tinyblob       TEXT
 317 old_text          mediumblob     TEXT
 318 pp_propname       varbinary(60)  TEXT
 319 pp_value          blob           TEXT
 320 page_restrictions tinyblob       TEXT # CSV string
 321 pf_server         varchar(30)    TEXT
 322 pr_level          varbinary(60)  TEXT
 323 pr_type           varbinary(60)  TEXT
 324 pt_create_perm    varbinary(60)  TEXT
 325 pt_reason         tinyblob       TEXT
 326 qc_type           varbinary(32)  TEXT
 327 qcc_type          varbinary(32)  TEXT
 328 qci_type          varbinary(32)  TEXT
 329 rc_params         blob           TEXT
 330 ug_group          varbinary(16)  TEXT
 331 user_email_token  binary(32)     TEXT
 332 user_ip           varbinary(40)  TEXT
 333 user_newpassword  tinyblob       TEXT
 334 user_options      blob           TEXT
 335 user_password     tinyblob       TEXT
 336 user_token        binary(32)     TEXT
 337
 338 ## Text URLs:
 339 el_index blob           TEXT
 340 el_to    blob           TEXT
 341 iw_url   blob           TEXT
 342 tb_url   blob           TEXT
 343 tc_url   varbinary(255) TEXT
 344
 345 ## Deprecated or not yet used:
 346 ar_text     mediumblob TEXT
 347 job_params  blob       TEXT
 348 log_deleted tinyint    INTEGER # Not used yet, but keep it INTEGER for safety
 349 rc_type     tinyint    CHAR
 350
 351 ## Number tweaking:
 352 fa_bits   int SMALLINT # bits per pixel
 353 fa_height int SMALLINT
 354 fa_width  int SMALLINT # Hope we don't see an image this wide...
 355 hc_id     int BIGINT   # Odd that site_stats is all bigint...
 356 img_bits  int SMALLINT # bits per image should stay sane
 357 oi_bits   int SMALLINT
 358
 359 ## True binary fields, usually due to gzdeflate and/or serialize:
 360 math_inputhash  varbinary(16) BYTEA
 361 math_outputhash varbinary(16) BYTEA
 362
 363 ## Namespaces: not need for such a high range
 364 ar_namespace   int SMALLINT
 365 job_namespace  int SMALLINT
 366 log_namespace  int SMALLINT
 367 page_namespace int SMALLINT
 368 pl_namespace   int SMALLINT
 369 pt_namespace   int SMALLINT
 370 qc_namespace   int SMALLINT
 371 rc_namespace   int SMALLINT
 372 rd_namespace   int SMALLINT
 373 tl_namespace   int SMALLINT
 374 wl_namespace   int SMALLINT
 375
 376 ## "Bools"
 377 ar_minor_edit    tinyint CHAR
 378 iw_trans         tinyint CHAR
 379 page_is_new      tinyint CHAR
 380 page_is_redirect tinyint CHAR
 381 rc_bot           tinyint CHAR
 382 rc_deleted       tinyint CHAR
 383 rc_minor         tinyint CHAR
 384 rc_new           tinyint CHAR
 385 rc_patrolled     tinyint CHAR
 386 rev_deleted      tinyint CHAR
 387 rev_minor_edit   tinyint CHAR
 388
 389 ## Easy enough to change if a wiki ever does grow this big:
 390 ss_good_articles bigint INTEGER
 391 ss_total_edits   bigint INTEGER
 392 ss_total_pages   bigint INTEGER
 393 ss_total_views   bigint INTEGER
 394 ss_users         bigint INTEGER
 395
 396 ## True IP - keep an eye on these, coders tend to make textual assumptions
 397 rc_ip varbinary(40) CIDR # Want to keep an eye on this
 398
 399 ## Others:
 400 tc_time int TIMESTAMPTZ
 401
 402
 403 };
 404
 405 my %colmap;
 406 for (split /\n/ => $COLMAP) {
 407         next unless /^\w/;
 408         s/(.*?)#.*/$1/;
 409         my ($col,@maps) = split / +/, $_;
 410         for (@maps) {
 411                 $colmap{$col}{$_} = 1;
 412         }
 413 }
 414
 415 my %colmapok;
 416 for (split /\n/ => $COLMAPOK) {
 417         next unless /^\w/;
 418         my ($col,$old,$new) = split / +/, $_;
 419         $colmapok{$col}{$old}{$new} = 1;
 420 }
 421
 422 ## Old but not new
 423 for my $t (sort keys %{$old{$oldfile}}) {
 424         if (!exists $new{$t} and !exists $ok{OLD}{$t}) {
 425                 print "Table not in $new: $t\n";
 426                 next;
 427         }
 428         next if exists $ok{OLD}{$t} and !$ok{OLD}{$t};
 429         my $newt = exists $ok{OLD}{$t} ? $ok{OLD}{$t} : $t;
 430         my $oldcol = $old{$oldfile}{$t}{column};
 431         my $oldcolfull = $old{$oldfile}{$t}{columnfull};
 432         my $newcol = $new{$newt}{column};
 433         for my $c (keys %$oldcol) {
 434                 if (!exists $newcol->{$c}) {
 435                         print "Column $t.$c not in $new\n";
 436                         next;
 437                 }
 438         }
 439         for my $c (sort keys %$newcol) {
 440                 if (!exists $oldcol->{$c}) {
 441                         print "Column $t.$c not in $oldfile\n";
 442                         next;
 443                 }
 444                 ## Column types (roughly) match up?
 445                 my $new = $newcol->{$c};
 446                 my $old = $oldcolfull->{$c};
 447
 448                 ## Known exceptions:
 449                 next if exists $colmapok{$c}{$old}{$new};
 450
 451                 $old =~ s/ENUM.*/ENUM/;
 452                 if (! exists $colmap{$old}{$new}) {
 453                         print "Column types for $t.$c do not match: $old does not map to $new\n";
 454                 }
 455         }
 456 }
 457 ## New but not old:
 458 for (sort keys %new) {
 459         if (!exists $old{$oldfile}{$_} and !exists $ok{NEW}{$_}) {
 460                 print "Not in $oldfile: $_\n";
 461                 next;
 462         }
 463 }
 464
 465
 466 } ## end each file to be parsed
 467
 468
 469 __DATA__
 470 ## Known exceptions
 471 OLD: searchindex          ## We use tsearch2 directly on the page table instead
 472 RENAME: user mwuser       ## Reserved word causing lots of problems
 473 RENAME: text pagecontent  ## Reserved word
 474 NEW: mediawiki_version    ## Just us, for now
 475 XFILE: ../archives/patch-profiling.sql