bkuhn/small-hacks
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support Spam Assassin; begin debug of memory leak.

Browse source 
I added support for spam assassin, but when I tried to run the script as in
on my mail server, it eats up 512MB/ram just processing the first 50 emails
or so.  I can't figure out why that is.
This commit is contained in:
Bradley M. Kuhn 2018-04-25 13:09:12 -07:00
parent f96f9242c7
commit 7b90af0611
1 changed files with 49 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
remove-spam-high-confidence-maildir.plx
View file

@ -22,17 +22,18 @@ use warnings;
use Mail::Header; use Mail::Header;
use Date::Manip; use Date::Manip;
use autodie qw(open close);
#use File::Copy; #use File::Copy;
my $VERBOSE = 1; my $VERBOSE = 1;
if (@ARGV < 4 or @ARGV > 5) { if (@ARGV < 5 or @ARGV > 6) {
print STDERR "usage: $0 <MAILDIR_DIRECTORY> <DSPAM_PROBABILITY_MIN> <DSPAM_CONFIDENCE_LEVEL_MIN> <DAYS> [<COUNT_ONLY_DONT_DELETE>]\n"; print STDERR "usage: $0 <MAILDIR_DIRECTORY> <DSPAM_PROBABILITY_MIN> <DSPAM_CONFIDENCE_LEVEL_MIN> <SPAM_ASSASSIN_SCORE> <DAYS> [<COUNT_ONLY_DONT_DELETE>]\n";
exit 1; exit 1;
} }
my($MAILDIR_FOLDER, $DSPAM_PROB_MIN, $DSPAM_CONF_MIN, $DAYS, $COUNT_ONLY) = @ARGV; my($MAILDIR_FOLDER, $DSPAM_PROB_MIN, $DSPAM_CONF_MIN, $SPAM_ASSASSIN_SCORE, $DAYS, $COUNT_ONLY) = @ARGV;
my($total, $countDeleted, $totalInDate) = (0, 0, 0); my($total, $countDeleted, $totalInDate) = (0, 0, 0);
@ -46,16 +47,17 @@ foreach my $dir (@msgDirs) {
} }
foreach my $dir (@msgDirs) { foreach my $dir (@msgDirs) {
opendir(MAILDIR, $dir) or die "Unable to open directory $dir for reading: $!"; opendir(MAILDIR, $dir) or die "Unable to open directory $dir for reading: $!";
MAIL: while (my $file = readdir MAILDIR) { while (my $file = readdir MAILDIR) {
print STDERR ".";
next if -d $file; # skip directories next if -d $file; # skip directories
my $fullFileName = "$dir/$file"; my $fullFileName = "$dir/$file";
unless (open(MAIL_MESSAGE, "<", $fullFileName)) { my $fh;
print STDERR "File, $fullFileName, appears to have disappeared during processing ($!).\n (Ignoring that fact, but counts may be off.)\n"; unless (open($fh, "<", $fullFileName)) {
next MAIL; print STDERR "File, $fullFileName, appears to have disappeared during processing ($!).\n";
} }
my $header = new Mail::Header(\*MAIL_MESSAGE); my $header = new Mail::Header($fh);
my $fields = $header->header_hashref; my $fields = $header->header_hashref;
my $mailDate; my $mailDate;
@ -68,18 +70,18 @@ MAIL: while (my $file = readdir MAILDIR) {
} }
if (not defined $mailDate) { if (not defined $mailDate) {
print STDERR "File $file has no Date: header. Skipping.\n"; print STDERR "File $file has no Date: header. Skipping.\n";
next MAIL; next;
} }
my $parsedDate = ParseDate($mailDate); my $parsedDate = ParseDate($mailDate);
unless (defined $parsedDate) { unless (defined $parsedDate) {
print STDERR "File $file has Unparsable Date header $mailDate"; print STDERR "File $file has Unparsable Date header $mailDate";
next MAIL; next;
} }
$total++; $total++;
print "\nDate: $parsedDate" if ($VERBOSE > 2); print "\nDate: $parsedDate" if ($VERBOSE > 2);
next MAIL if ($parsedDate gt $nDaysAgo); next if ($parsedDate gt $nDaysAgo);
$totalInDate++; $totalInDate++;
print " Not skipping over date, $nDaysAgo\n" if ($VERBOSE > 2); print " Not skipping over date, $nDaysAgo\n" if ($VERBOSE > 2);
@ -93,27 +95,50 @@ MAIL: while (my $file = readdir MAILDIR) {
$dspamVal{$val} = $dv if $dv < $dspamVal{$val}; $dspamVal{$val} = $dv if $dv < $dspamVal{$val};
} }
} }
if (not defined $dspamVal{$val}) {
print STDERR "File $file has no X-Dspam-$val header. Skipping.\n";
next MAIL;
}
} }
print " Confidence: $dspamVal{Confidence}, Probability: $dspamVal{Probability}\n" print " Confidence: $dspamVal{Confidence}, Probability: $dspamVal{Probability}\n"
if ($VERBOSE > 2); if ($VERBOSE > 2);
my $isReadyToDelete = 0;
if (defined $dspamVal{Confidence} and defined $dspamVal{Probability}) {
$isReadyToDelete = 1
if ($dspamVal{Confidence} >= $DSPAM_CONF_MIN and
$dspamVal{Probability} >= $DSPAM_PROB_MIN);
if ($dspamVal{Confidence} >= $DSPAM_CONF_MIN and } else {
$dspamVal{Probability} >= $DSPAM_PROB_MIN) { my $spamStatusVal;
$countDeleted++; foreach my $dv (@{$fields->{"X-Spam-Status"}}) {
print " counting this one\n" if ($VERBOSE > 2); chomp $dv;
unless (defined $COUNT_ONLY and $COUNT_ONLY) { print " X-Spam-Status found: $dv\n" if ($VERBOSE > 3);
warn "unable to unlink $fullFileName: $!" if ($dv =~ /Yes.*score\s*=\s*([\d\.]+)\s+/i) {
unless unlink("$fullFileName") == 1; my $newVal = $1;
if (not defined $spamStatusVal) {
$spamStatusVal = $newVal;
} else {
$spamStatusVal = $newVal if $newVal < $spamStatusVal;
}
}
}
print " Final Spam Status from Spam Assassin: $spamStatusVal\n" if ($VERBOSE > 2);
if (not defined $spamStatusVal) {
print STDERR "File $file has no headers for Spam. Skipping.\n";
} else {
$isReadyToDelete = 1 if ($spamStatusVal >= $SPAM_ASSASSIN_SCORE);
}
if ($isReadyToDelete) {
$countDeleted++;
print " counting this one\n" if ($VERBOSE > 2);
unless (defined $COUNT_ONLY and $COUNT_ONLY) {
warn "unable to unlink $fullFileName: $!"
unless unlink("$fullFileName") == 1;
}
} }
} }
close MAIL_MESSAGE; $fh->close();
$fh = $header = $fields = undef;
} }
close MAILDIR; close MAILDIR;
print STDERR ".\n";
} }
my $percent = ($countDeleted / $total) * 100.00; my $percent = ($countDeleted / $total) * 100.00;