#!/usr/bin/perl -w
# streamlog.pl - v1.0
#    Automatically filter the latest rotated compressed Web server log
#    file as it streams from the FTP site, using no disk and allowing
#    for bandwidth conservation.
#
#    It assumes that the files are stored in such a way that alphabetically
#    sorted, the files matching the regexp will show up with the latest last.
#
#    StreamLog is distributed under version 2.0 or greater of the GNU GPL.
#
#    vsync@quadium.net
#    http://quadium.net/

use strict;
use Net::FTP;
use IPC::Open2;

my $server = "my.web.site"; # web server
my $logdir = "/usr/local/apache/logs";    # where the logs are kept
my $user = "login";
my $pass = 'pass';
my $logpat = "access_log\..*\.gz"; # regexp to match compressed logs with
my $sitepat = "my\.web\.site"; # regexp to match site log lines with
my $loglength = 60*60*24; # maximum delay allowed between creation and reading
                          # this should be set <= the interval this is run
                          # I run it daily from cron
my $finalout = "|/var/lib/apache/sbin/logresolve >> /var/lib/apache/htdocs/usage/access_log"; # what to do with the final output

print "Connecting...\n";
my $ftp = Net::FTP->new($server) or die "Couldn't connect";
print "Logging in...\n";
$ftp->login($user, $pass);
print "Setting working directory...\n";
$ftp->cwd($logdir);
print "Listing files...\n";
my @dirlist = $ftp->ls();
my @logs;

my $i;
foreach $i (@dirlist) {
  if ($i =~ /$logpat/) {
    push @logs, $i;
  }
}

# Get the last file and make sure we haven't seen this already
@logs = sort @logs;
my $file = pop @logs;
my $modtime = $ftp->mdtm($file);
if ((time - $modtime) > $loglength) {
  my $pretty = gmtime($modtime);
  print STDERR "$file is too old (",$pretty,"), skipping...\n";
  $ftp->quit;
  exit 1;
}

print "Streaming $file...\n";

my $tempfile = "/tmp/streamlog.$$";
system("mkfifo $tempfile");

if (fork) {                     # half of us will decompress
  open(*RAW, "|gzip -dc > $tempfile");
  $ftp->get($file, *RAW);
  close(*RAW);
  $ftp->quit;
} else {                        # the other half will filter
  open(*TEXT, $tempfile);
  open(*FINALOUT, $finalout);
  while (<TEXT>) {
    if (/$sitepat/) {
      print FINALOUT;
    }
  }
  close(*TEXT);
  close(FINALOUT);
  unlink($tempfile);
  print "Done!\n";
}