3 ## Rough check that the base and postgres "tables.sql" are in sync
4 ## Should be run from maintenance/postgres
5 ## Checks a few other things as well...
13 my @old = ('../tables.sql');
14 my $new = 'tables.sql';
17 ## Read in exceptions and other metadata
20 next unless /^(\w+)\s*:\s*([^#]+)/;
21 my ($name,$val) = ($1,$2);
23 if ($name eq 'RENAME') {
24 die "Invalid rename\n" unless $val =~ /(\w+)\s+(\w+)/;
29 if ($name eq 'XFILE') {
33 for (split /\s+/ => $val) {
38 my $datatype = join '|' => qw(
40 tinyint int bigint real float
41 tinytext mediumtext text char varchar varbinary binary
43 tinyblob mediumblob blob
45 $datatype .= q{|ENUM\([\"\w, ]+\)};
46 $datatype = qr{($datatype)};
48 my $typeval = qr{(\(\d+\))?};
50 my $typeval2 = qr{ signed| unsigned| binary| NOT NULL| NULL| auto_increment| default ['\-\d\w"]+| REFERENCES .+CASCADE};
52 my $indextype = join '|' => qw(INDEX KEY FULLTEXT), 'PRIMARY KEY', 'UNIQUE INDEX', 'UNIQUE KEY';
53 $indextype = qr{$indextype};
55 my $engine = qr{TYPE|ENGINE};
57 my $tabletype = qr{InnoDB|MyISAM|HEAP|HEAP MAX_ROWS=\d+|InnoDB MAX_ROWS=\d+ AVG_ROW_LENGTH=\d+};
59 my $charset = qr{utf8|binary};
61 open my $newfh, '<', $new or die qq{Could
not open $new: $!\n};
68 for my $xfile (@xfile) {
69 print "Loading $xfile\n";
70 my $info = parse_sql
($xfile);
72 $xinfo{$_} = $info->{$_};
76 for my $oldfile (@old) {
77 print "Loading $oldfile\n";
78 my $info = parse_sql
($oldfile);
80 $info->{$_} = $xinfo{$_};
82 $old{$oldfile} = $info;
89 open my $oldfh, '<', $oldfile or die qq{Could
not open $oldfile: $!\n};
93 next if /^\s*\-\-/ or /^\s+$/;
97 if (/CREATE\s*TABLE/i) {
98 m{^CREATE TABLE /\*\$wgDBprefix\*/(\w+) \($}
99 or die qq{Invalid CREATE TABLE at line
$. of
$oldfile\n};
101 $info{$table}{name
}=$table;
103 elsif (m{^\) /\*\$wgDBTableOptions\*/}) {
104 $info{$table}{engine
} = 'TYPE';
105 $info{$table}{type
} = 'variable';
107 elsif (/^\) ($engine)=($tabletype);$/) {
108 $info{$table}{engine
}=$1;
109 $info{$table}{type
}=$2;
111 elsif (/^\) ($engine)=($tabletype), DEFAULT CHARSET=($charset);$/) {
112 $info{$table}{engine
}=$1;
113 $info{$table}{type
}=$2;
114 $info{$table}{charset
}=$3;
116 elsif (/^ (\w+) $datatype$typeval$typeval2{0,3},?$/) {
117 $info{$table}{column
}{$1} = $2;
118 my $extra = $3 || '';
119 $info{$table}{columnfull
}{$1} = "$2$extra";
121 elsif (/^ ($indextype)(?: (\w+))? \(([\w, \(\)]+)\),?$/) {
122 $info{$table}{lc $1.'_name'} = $2 ?
$2 : '';
123 $info{$table}{lc $1.'pk_target'} = $3;
126 die "Cannot parse line $. of $oldfile:\n$_\n";
130 close $oldfh or die qq{Could
not close "$oldfile": $!\n};
134 } ## end of parse_sql
136 ## Read in the parser test information
137 my $parsefile = '../parserTests.inc';
138 open my $pfh, '<', $parsefile or die qq{Could
not open "$parsefile": $!\n};
143 if (/function listTables/) {
148 $ptable{$1}=2 while m{'(\w+)'}g;
151 close $pfh or die qq{Could
not close "$parsefile": $!\n};
153 my $OK_NOT_IN_PTABLE = '
165 ## Make sure all tables in main tables.sql are accounted for int the parsertest.
166 for my $table (sort keys %{$old{'../tables.sql'}}) {
168 next if $ptable{$table} > 2;
169 next if $OK_NOT_IN_PTABLE =~ /\b$table\b/;
170 print qq{Table
"$table" is
in the schema
, but
not used inside of parserTest
.inc
\n};
172 ## Any that are used in ptables but no longer exist in the schema?
173 for my $table (sort grep { $ptable{$_} == 2 } keys %ptable) {
174 print qq{Table
"$table" ($ptable{$table}) used
in parserTest
.inc
, but
not found
in schema
\n};
177 for my $oldfile (@old) {
179 ## Begin non-standard indent
181 ## MySQL sanity checks
182 for my $table (sort keys %{$old{$oldfile}}) {
183 my $t = $old{$oldfile}{$table};
184 if (($oldfile =~ /5/ and $t->{engine
} ne 'ENGINE')
186 ($oldfile !~ /5/ and $t->{engine
} ne 'TYPE')) {
187 die "Invalid engine for $oldfile: $t->{engine}\n" unless $t->{name
} eq 'profiling';
189 my $charset = $t->{charset
} || '';
190 if ($oldfile !~ /binary/ and $charset eq 'binary') {
191 die "Invalid charset for $oldfile: $charset\n";
195 my $dtype = join '|' => qw(
196 SMALLINT INTEGER BIGINT NUMERIC SERIAL
202 $dtype = qr{($dtype)};
204 my ($infunction,$inview,$inrule,$lastcomma) = (0,0,0,0);
207 next if /^\s*\-\-/ or /^\s*$/;
208 s/\s*\-\- [\w ']+$//;
209 next if /^BEGIN;/ or /^SET / or /^COMMIT;/;
210 next if /^CREATE SEQUENCE/;
211 next if /^CREATE(?: UNIQUE)? INDEX/;
212 next if /^CREATE FUNCTION/;
213 next if /^CREATE TRIGGER/ or /^ FOR EACH ROW/;
214 next if /^INSERT INTO/ or /^ VALUES \(/;
215 next if /^ALTER TABLE/;
219 $infunction = $infunction ?
0 : 1;
224 next if /^CREATE VIEW/ and $inview = 1;
226 /;$/ and $inview = 0;
230 next if /^CREATE RULE/ and $inrule = 1;
232 /;$/ and $inrule = 0;
236 if (/^CREATE TABLE "?(\w+)"? \($/) {
238 $new{$table}{name
}=$table;
243 warn "Stray comma before line $.\n";
246 elsif (/^ (\w+) +$dtype.*?(,?)(?: --.*)?$/) {
247 $new{$table}{column
}{$1} = $2;
249 print "Missing comma before line $. of $new\n";
251 $lastcomma = $3 ?
1 : 0;
254 die "Cannot parse line $. of $new:\n$_\n";
258 ## Which column types are okay to map from mysql to postgres?
278 varbinary(14) TIMESTAMPTZ
279 binary(14) TIMESTAMPTZ
281 timestamp TIMESTAMPTZ
290 ## Allow specific exceptions to the above
292 ## User inputted text strings:
293 ar_comment tinyblob TEXT
294 fa_description tinyblob TEXT
295 img_description tinyblob TEXT
296 ipb_reason tinyblob TEXT
297 log_action varbinary(10) TEXT
298 oi_description tinyblob TEXT
299 rev_comment tinyblob TEXT
300 rc_log_action varbinary(255) TEXT
301 rc_log_type varbinary(255) TEXT
303 ## Simple text-only strings:
304 ar_flags tinyblob TEXT
305 fa_minor_mime varbinary(32) TEXT
306 fa_storage_group varbinary(16) TEXT # Just 'deleted' for now, should stay plain text
307 fa_storage_key varbinary(64) TEXT # sha1 plus text extension
308 ipb_address tinyblob TEXT # IP address or username
309 ipb_range_end tinyblob TEXT # hexadecimal
310 ipb_range_start tinyblob TEXT # hexadecimal
311 img_minor_mime varbinary(32) TEXT
312 img_sha1 varbinary(32) TEXT
313 job_cmd varbinary(60) TEXT # Should we limit to 60 as well?
314 keyname varbinary(255) TEXT # No tablename prefix (objectcache)
315 ll_lang varbinary(20) TEXT # Language code
316 log_params blob TEXT # LF separated list of args
317 log_type varbinary(10) TEXT
318 oi_minor_mime varbinary(32) TEXT
319 oi_sha1 varbinary(32) TEXT
320 old_flags tinyblob TEXT
321 old_text mediumblob TEXT
322 pp_propname varbinary(60) TEXT
324 page_restrictions tinyblob TEXT # CSV string
325 pf_server varchar(30) TEXT
326 pr_level varbinary(60) TEXT
327 pr_type varbinary(60) TEXT
328 pt_create_perm varbinary(60) TEXT
329 pt_reason tinyblob TEXT
330 qc_type varbinary(32) TEXT
331 qcc_type varbinary(32) TEXT
332 qci_type varbinary(32) TEXT
334 rlc_to_blob blob TEXT
335 ug_group varbinary(16) TEXT
336 user_email_token binary(32) TEXT
337 user_ip varbinary(40) TEXT
338 user_newpassword tinyblob TEXT
339 user_options blob TEXT
340 user_password tinyblob TEXT
341 user_token binary(32) TEXT
348 tc_url varbinary(255) TEXT
350 ## Deprecated or not yet used:
351 ar_text mediumblob TEXT
353 log_deleted tinyint INTEGER # Not used yet, but keep it INTEGER for safety
357 fa_bits int SMALLINT # bits per pixel
358 fa_height int SMALLINT
359 fa_width int SMALLINT # Hope we don't see an image this wide...
360 hc_id int BIGINT # Odd that site_stats is all bigint...
361 img_bits int SMALLINT # bits per image should stay sane
364 ## True binary fields, usually due to gzdeflate and/or serialize:
365 math_inputhash varbinary(16) BYTEA
366 math_outputhash varbinary(16) BYTEA
368 ## Namespaces: not need for such a high range
369 ar_namespace int SMALLINT
370 job_namespace int SMALLINT
371 log_namespace int SMALLINT
372 page_namespace int SMALLINT
373 pl_namespace int SMALLINT
374 pt_namespace int SMALLINT
375 qc_namespace int SMALLINT
376 rc_namespace int SMALLINT
377 rd_namespace int SMALLINT
378 rlc_to_namespace int SMALLINT
379 tl_namespace int SMALLINT
380 wl_namespace int SMALLINT
382 ## Easy enough to change if a wiki ever does grow this big:
383 ss_good_articles bigint INTEGER
384 ss_total_edits bigint INTEGER
385 ss_total_pages bigint INTEGER
386 ss_total_views bigint INTEGER
387 ss_users bigint INTEGER
389 ## True IP - keep an eye on these, coders tend to make textual assumptions
390 rc_ip varbinary(40) CIDR # Want to keep an eye on this
393 tc_time int TIMESTAMPTZ
399 for (split /\n/ => $COLMAP) {
402 my ($col,@maps) = split / +/, $_;
404 $colmap{$col}{$_} = 1;
409 for (split /\n/ => $COLMAPOK) {
411 my ($col,$old,$new) = split / +/, $_;
412 $colmapok{$col}{$old}{$new} = 1;
416 for my $t (sort keys %{$old{$oldfile}}) {
417 if (!exists $new{$t} and !exists $ok{OLD
}{$t}) {
418 print "Table not in $new: $t\n";
421 next if exists $ok{OLD
}{$t} and !$ok{OLD
}{$t};
422 my $newt = exists $ok{OLD
}{$t} ?
$ok{OLD
}{$t} : $t;
423 my $oldcol = $old{$oldfile}{$t}{column
};
424 my $oldcolfull = $old{$oldfile}{$t}{columnfull
};
425 my $newcol = $new{$newt}{column
};
426 for my $c (keys %$oldcol) {
427 if (!exists $newcol->{$c}) {
428 print "Column $t.$c not in $new\n";
432 for my $c (sort keys %$newcol) {
433 if (!exists $oldcol->{$c}) {
434 print "Column $t.$c not in $oldfile\n";
437 ## Column types (roughly) match up?
438 my $new = $newcol->{$c};
439 my $old = $oldcolfull->{$c};
442 next if exists $colmapok{$c}{$old}{$new};
444 $old =~ s/ENUM.*/ENUM/;
445 if (! exists $colmap{$old}{$new}) {
446 print "Column types for $t.$c do not match: $old does not map to $new\n";
451 for (sort keys %new) {
452 if (!exists $old{$oldfile}{$_} and !exists $ok{NEW
}{$_}) {
453 print "Not in $oldfile: $_\n";
459 } ## end each file to be parsed
462 sub check_includes_dir
{
464 ## Check for some common errors in the files in the includes directory
466 print "Checking files in includes directory...\n";
467 my $dir = '../../includes';
468 opendir my $dh, $dir or die qq{Could
not opendir $dir: $!\n};
469 for my $file (grep { -f
"$dir/$_" and /\
.php
$/ } readdir $dh) {
470 $file = "$dir/$file";
471 open my $fh, '<', $file or die qq{Could
not open "$file": $!\n};
473 if (/FORCE INDEX/ and $file !~ /Database.php/) {
474 warn "Found FORCE INDEX string at line $. of $file\n";
476 if (/REPLACE INTO/ and $file !~ /Database/) {
477 warn "Found REPLACE INTO string at line $. of $file\n";
479 if (/\bIF\s*\(/ and $file !~ /Database.php/) {
480 warn "Found IF string at line $. of $file\n";
482 if (/\bCONCAT\b/ and $file !~ /Database.php/) {
483 warn "Found CONCAT string at line $. of $file\n";
486 close $fh or die qq{Could
not close "$file": $!\n};
488 closedir $dh or die qq{Closedir failed?
!\n};
492 } ## end of check_includes_dir
496 OLD
: searchindex
## We use tsearch2 directly on the page table instead
497 RENAME
: user mwuser
## Reserved word causing lots of problems
498 RENAME
: text pagecontent
## Reserved word
499 NEW
: mediawiki_version
## Just us, for now
500 XFILE
: ../archives/patch
-profiling
.sql