3 ## Rough check that the base and postgres "tables.sql" are in sync
4 ## Should be run from maintenance/postgres
5 ## Checks a few other things as well...
14 my @old = ('../tables.sql');
15 my $new = 'tables.sql';
18 ## Read in exceptions and other metadata
21 next unless /^(\w+)\s*:\s*([^#]+)/;
22 my ($name,$val) = ($1,$2);
24 if ($name eq 'RENAME') {
25 die "Invalid rename\n" unless $val =~ /(\w+)\s+(\w+)/;
30 if ($name eq 'XFILE') {
34 for (split /\s+/ => $val) {
39 my $datatype = join '|' => qw(
41 tinyint int bigint real float
42 tinytext mediumtext text char varchar varbinary binary
44 tinyblob mediumblob blob
46 $datatype .= q{|ENUM\([\"\w\', ]+\)};
47 $datatype = qr{($datatype)};
49 my $typeval = qr{(\(\d+\))?};
51 my $typeval2 = qr{ signed| unsigned| binary| NOT NULL| NULL| PRIMARY KEY| AUTO_INCREMENT| default ['\-\d\w"]+| REFERENCES .+CASCADE};
53 my $indextype = join '|' => qw(INDEX KEY FULLTEXT), 'PRIMARY KEY', 'UNIQUE INDEX', 'UNIQUE KEY';
54 $indextype = qr{$indextype};
56 my $engine = qr{TYPE|ENGINE};
58 my $tabletype = qr{InnoDB|MyISAM|HEAP|HEAP MAX_ROWS=\d+|InnoDB MAX_ROWS=\d+ AVG_ROW_LENGTH=\d+};
60 my $charset = qr{utf8|binary};
62 open my $newfh, '<', $new or die qq{Could
not open $new: $!\n};
69 for my $xfile (@xfile) {
70 print "Loading $xfile\n";
71 my $info = parse_sql
($xfile);
73 $xinfo{$_} = $info->{$_};
77 for my $oldfile (@old) {
78 print "Loading $oldfile\n";
79 my $info = parse_sql
($oldfile);
81 $info->{$_} = $xinfo{$_};
83 $old{$oldfile} = $info;
90 open my $oldfh, '<', $oldfile or die qq{Could
not open $oldfile: $!\n};
94 next if /^\s*\-\-/ or /^\s+$/;
98 if (/CREATE\s*TABLE/i) {
99 if (m{^CREATE TABLE /\*_\*/(\w+) \($}) {
102 elsif (m{^CREATE TABLE /\*\$wgDBprefix\*/(\w+) \($}) {
106 die qq{Invalid CREATE TABLE at line
$. of
$oldfile\n};
108 $info{$table}{name
}=$table;
110 elsif (m{^\) /\*\$wgDBTableOptions\*/}) {
111 $info{$table}{engine
} = 'ENGINE';
112 $info{$table}{type
} = 'variable';
114 elsif (/^\) ($engine)=($tabletype);$/) {
115 $info{$table}{engine
}=$1;
116 $info{$table}{type
}=$2;
118 elsif (/^\) ($engine)=($tabletype), DEFAULT CHARSET=($charset);$/) {
119 $info{$table}{engine
}=$1;
120 $info{$table}{type
}=$2;
121 $info{$table}{charset
}=$3;
123 elsif (/^ (\w+) $datatype$typeval$typeval2{0,4},?$/) {
124 $info{$table}{column
}{$1} = $2;
125 my $extra = $3 || '';
126 $info{$table}{columnfull
}{$1} = "$2$extra";
128 elsif (m{^ UNIQUE KEY (\w+) \((.+?)\)}) {
130 elsif (m{^CREATE (?:UNIQUE )?(?:FULLTEXT )?INDEX /\*i\*/(\w+) ON /\*_\*/(\w+) \((.+?)\);}) {
132 elsif (m{^\s*PRIMARY KEY \([\w,]+\)}) {
135 die "Cannot parse line $. of $oldfile:\n$_\n";
139 close $oldfh or die qq{Could
not close "$oldfile": $!\n};
143 } ## end of parse_sql
145 ## Read in the parser test information
146 my $parsefile = '../parserTests.inc';
147 open my $pfh, '<', $parsefile or die qq{Could
not open "$parsefile": $!\n};
152 if (/function listTables/) {
157 $ptable{$1}=2 while m{'(\w+)'}g;
160 close $pfh or die qq{Could
not close "$parsefile": $!\n};
162 my $OK_NOT_IN_PTABLE = '
177 ## Make sure all tables in main tables.sql are accounted for in the parsertest.
178 for my $table (sort keys %{$old{'../tables.sql'}}) {
180 next if $ptable{$table} > 2;
181 next if $OK_NOT_IN_PTABLE =~ /\b$table\b/;
182 print qq{Table
"$table" is
in the schema
, but
not used inside of parserTest
.inc
\n};
184 ## Any that are used in ptables but no longer exist in the schema?
185 for my $table (sort grep { $ptable{$_} == 2 } keys %ptable) {
186 print qq{Table
"$table" ($ptable{$table}) used
in parserTest
.inc
, but
not found
in schema
\n};
189 for my $oldfile (@old) {
191 ## Begin non-standard indent
193 ## MySQL sanity checks
194 for my $table (sort keys %{$old{$oldfile}}) {
195 my $t = $old{$oldfile}{$table};
196 if ($t->{engine
} eq 'TYPE') {
197 die "Invalid engine for $oldfile: $t->{engine}\n" unless $t->{name
} eq 'profiling';
199 my $charset = $t->{charset
} || '';
200 if ($oldfile !~ /binary/ and $charset eq 'binary') {
201 die "Invalid charset for $oldfile: $charset\n";
205 my $dtype = join '|' => qw(
206 SMALLINT INTEGER BIGINT NUMERIC SERIAL
212 $dtype = qr{($dtype)};
214 my ($infunction,$inview,$inrule,$lastcomma) = (0,0,0,0);
217 next if /^\s*\-\-/ or /^\s*$/;
218 s/\s*\-\- [\w ']+$//;
219 next if /^BEGIN;/ or /^SET / or /^COMMIT;/;
220 next if /^CREATE SEQUENCE/;
221 next if /^CREATE(?: UNIQUE)? INDEX/;
222 next if /^CREATE FUNCTION/;
223 next if /^CREATE TRIGGER/ or /^ FOR EACH ROW/;
224 next if /^INSERT INTO/ or /^ VALUES \(/;
225 next if /^ALTER TABLE/;
229 $infunction = $infunction ?
0 : 1;
234 next if /^CREATE VIEW/ and $inview = 1;
236 /;$/ and $inview = 0;
240 next if /^CREATE RULE/ and $inrule = 1;
242 /;$/ and $inrule = 0;
246 if (/^CREATE TABLE "?(\w+)"? \($/) {
248 $new{$table}{name
}=$table;
253 warn "Stray comma before line $.\n";
256 elsif (/^ (\w+) +$dtype.*?(,?)(?: --.*)?$/) {
257 $new{$table}{column
}{$1} = $2;
259 print "Missing comma before line $. of $new\n";
261 $lastcomma = $3 ?
1 : 0;
263 elsif (m{^\s*PRIMARY KEY \([\w,]+\)}) {
267 die "Cannot parse line $. of $new:\n$_\n";
271 ## Which column types are okay to map from mysql to postgres?
291 varbinary(14) TIMESTAMPTZ
292 binary(14) TIMESTAMPTZ
294 timestamp TIMESTAMPTZ
303 ## Allow specific exceptions to the above
305 ## User inputted text strings:
306 ar_comment tinyblob TEXT
307 fa_description tinyblob TEXT
308 img_description tinyblob TEXT
309 ipb_reason tinyblob TEXT
310 log_action varbinary(32) TEXT
311 log_type varbinary(32) TEXT
312 oi_description tinyblob TEXT
313 rev_comment tinyblob TEXT
314 rc_log_action varbinary(255) TEXT
315 rc_log_type varbinary(255) TEXT
317 ## Simple text-only strings:
318 ar_flags tinyblob TEXT
320 fa_minor_mime varbinary(100) TEXT
321 fa_storage_group varbinary(16) TEXT # Just 'deleted' for now, should stay plain text
322 fa_storage_key varbinary(64) TEXT # sha1 plus text extension
323 ipb_address tinyblob TEXT # IP address or username
324 ipb_range_end tinyblob TEXT # hexadecimal
325 ipb_range_start tinyblob TEXT # hexadecimal
326 img_minor_mime varbinary(100) TEXT
327 lc_lang varbinary(32) TEXT
328 lc_value varbinary(32) TEXT
329 img_sha1 varbinary(32) TEXT
330 iw_wikiid varchar(64) TEXT
331 job_cmd varbinary(60) TEXT # Should we limit to 60 as well?
332 keyname varbinary(255) TEXT # No tablename prefix (objectcache)
333 ll_lang varbinary(20) TEXT # Language code
334 lc_value mediumblob TEXT
335 log_params blob TEXT # LF separated list of args
336 log_type varbinary(10) TEXT
337 ls_field varbinary(32) TEXT
338 oi_minor_mime varbinary(100) TEXT
339 oi_sha1 varbinary(32) TEXT
340 old_flags tinyblob TEXT
341 old_text mediumblob TEXT
342 pp_propname varbinary(60) TEXT
344 page_restrictions tinyblob TEXT # CSV string
345 pf_server varchar(30) TEXT
346 pr_level varbinary(60) TEXT
347 pr_type varbinary(60) TEXT
348 pt_create_perm varbinary(60) TEXT
349 pt_reason tinyblob TEXT
350 qc_type varbinary(32) TEXT
351 qcc_type varbinary(32) TEXT
352 qci_type varbinary(32) TEXT
354 rlc_to_blob blob TEXT
356 ug_group varbinary(16) TEXT
358 up_property varbinary(32) TEXT
360 user_email_token binary(32) TEXT
361 user_ip varbinary(40) TEXT
362 user_newpassword tinyblob TEXT
363 user_options blob TEXT
364 user_password tinyblob TEXT
365 user_token binary(32) TEXT
366 iwl_prefix varbinary(20) TEXT
374 tc_url varbinary(255) TEXT
376 ## Deprecated or not yet used:
377 ar_text mediumblob TEXT
379 log_deleted tinyint INTEGER # Not used yet, but keep it INTEGER for safety
383 fa_bits int SMALLINT # bits per pixel
384 fa_height int SMALLINT
385 fa_width int SMALLINT # Hope we don't see an image this wide...
386 hc_id int BIGINT # Odd that site_stats is all bigint...
387 img_bits int SMALLINT # bits per image should stay sane
390 ## True binary fields, usually due to gzdeflate and/or serialize:
391 math_inputhash varbinary(16) BYTEA
392 math_outputhash varbinary(16) BYTEA
394 ## Namespaces: not need for such a high range
395 ar_namespace int SMALLINT
396 job_namespace int SMALLINT
397 log_namespace int SMALLINT
398 page_namespace int SMALLINT
399 pl_namespace int SMALLINT
400 pt_namespace int SMALLINT
401 qc_namespace int SMALLINT
402 rc_namespace int SMALLINT
403 rd_namespace int SMALLINT
404 rlc_to_namespace int SMALLINT
405 tl_namespace int SMALLINT
406 wl_namespace int SMALLINT
408 ## Easy enough to change if a wiki ever does grow this big:
409 ss_active_users bigint INTEGER
410 ss_good_articles bigint INTEGER
411 ss_total_edits bigint INTEGER
412 ss_total_pages bigint INTEGER
413 ss_total_views bigint INTEGER
414 ss_users bigint INTEGER
416 ## True IP - keep an eye on these, coders tend to make textual assumptions
417 rc_ip varbinary(40) CIDR # Want to keep an eye on this
420 tc_time int TIMESTAMPTZ
426 for (split /\n/ => $COLMAP) {
429 my ($col,@maps) = split / +/, $_;
431 $colmap{$col}{$_} = 1;
436 for (split /\n/ => $COLMAPOK) {
438 my ($col,$old,$new) = split / +/, $_;
439 $colmapok{$col}{$old}{$new} = 1;
443 for my $t (sort keys %{$old{$oldfile}}) {
444 if (!exists $new{$t} and !exists $ok{OLD
}{$t}) {
445 print "Table not in $new: $t\n";
448 next if exists $ok{OLD
}{$t} and !$ok{OLD
}{$t};
449 my $newt = exists $ok{OLD
}{$t} ?
$ok{OLD
}{$t} : $t;
450 my $oldcol = $old{$oldfile}{$t}{column
};
451 my $oldcolfull = $old{$oldfile}{$t}{columnfull
};
452 my $newcol = $new{$newt}{column
};
453 for my $c (keys %$oldcol) {
454 if (!exists $newcol->{$c}) {
455 print "Column $t.$c not in $new\n";
459 for my $c (sort keys %$newcol) {
460 if (!exists $oldcol->{$c}) {
461 print "Column $t.$c not in $oldfile\n";
464 ## Column types (roughly) match up?
465 my $new = $newcol->{$c};
466 my $old = $oldcolfull->{$c};
469 next if exists $colmapok{$c}{$old}{$new};
471 $old =~ s/ENUM.*/ENUM/;
472 if (! exists $colmap{$old}{$new}) {
473 print "Column types for $t.$c do not match: $old does not map to $new\n";
478 for (sort keys %new) {
479 if (!exists $old{$oldfile}{$_} and !exists $ok{NEW
}{$_}) {
480 print "Not in $oldfile: $_\n";
486 } ## end each file to be parsed
489 sub check_valid_sql
{
491 ## Check for a few common problems in most php files
493 my $olddir = getcwd
();
495 for my $basedir (qw
/includes extensions/) {
502 } ## end of check_valid_sql
509 opendir my $dh, $dir or die qq{Could
not opendir $dir: $!\n};
510 #print "Scanning $dir...\n";
511 for my $file (grep { -f
"$dir/$_" and /\
.php
$/ } readdir $dh) {
512 find_problems
("$dir/$file");
515 for my $subdir (grep { -d
"$dir/$_" and ! /\
./ } readdir $dh) {
516 scan_dir
("$dir/$subdir");
518 closedir $dh or die qq{Closedir failed
: $!\n};
526 open my $fh, '<', $file or die qq{Could
not open "$file": $!\n};
530 if (/FORCE INDEX/ and $file !~ /Database\w*\.php/) {
531 warn "Found FORCE INDEX string at line $. of $file\n";
533 if (/REPLACE INTO/ and $file !~ /Database\w*\.php/) {
534 warn "Found REPLACE INTO string at line $. of $file\n";
536 if (/\bIF\s*\(/ and $file !~ /DatabaseMySQL\.php/) {
537 warn "Found IF string at line $. of $file\n";
539 if (/\bCONCAT\b/ and $file !~ /Database\w*\.php/) {
540 warn "Found CONCAT string at line $. of $file\n";
542 if (/\bGROUP\s+BY\s*\d\b/i and $file !~ /Database\w*\.php/) {
543 warn "Found GROUP BY # at line $. of $file\n";
545 if (/wfGetDB\s*\(\s+\)/io) {
546 warn "wfGETDB is missing parameters at line $. of $file\n";
548 if (/=\s*array\s*\(\s*$/) {
557 next if ! /\w/ or /array\(\s*$/ or /^\s*#/ or m{^\s*//};
559 my $nextline = <$fh>;
560 last if ! defined $nextline;
561 if ($nextline =~ /^\s*\)[;,]/) {
565 #warn "Array is missing a comma? Line $. of $file\n";
569 close $fh or die qq{Could
not close "$file": $!\n};
572 } ## end of find_problems
577 OLD
: searchindex
## We use tsearch2 directly on the page table instead
578 RENAME
: user mwuser
## Reserved word causing lots of problems
579 RENAME
: text pagecontent
## Reserved word
580 XFILE
: ../archives/patch
-profiling
.sql