[LON-CAPA-cvs] cvs: modules /raeburn monitor.pl

raeburn lon-capa-cvs@mail.lon-capa.org
Mon, 28 Mar 2005 22:38:59 -0000


This is a MIME encoded message

--raeburn1112049539
Content-Type: text/plain

raeburn		Mon Mar 28 17:38:59 2005 EDT

  Added files:                 
    /modules/raeburn	monitor.pl 
  Log:
  Script to monitor login, authentication, roles display, course intialization and navmap  display in a LON-CAPA course. Sends e-mail to designated recipients in case of repeated failure at one of these steps. Intended to be run on a remote server, and called from /etc/cron.d.  Page load times could be stored and displayed via MRTG.   
  
  
--raeburn1112049539
Content-Type: text/plain
Content-Disposition: attachment; filename="raeburn-20050328173859.txt"


Index: modules/raeburn/monitor.pl
+++ modules/raeburn/monitor.pl
#! /usr/bin/perl
use strict;
use lib qw(/usr/lib/perl5/site_perl/);
use HTTP::Request::Common;
use HTTP::Cookies;
use LWP::UserAgent;
use Crypt::CBC;
use Time::HiRes qw(gettimeofday tv_interval);

#########################################################
# monitor.pl
#
# Stuart Raeburn, March 28, 2005
#
#########################################################
# 
# Script to monitor log-in to LON-CAPA production servers.
# Specify the directory where monitoring data will be stored, by setting
# $monitordir.
# For each server that is to be monitored, a subdirectory should be created
# in $monitordir using a short (unique name), e.g., s1 for s1.lite.msu.edu, 
# s2 for s2.lite.msu.edu etc.
#
# An autouser.dat file should be placed in each server subdirectory.
# The autouser.dat file will contain username, domain, course role,
# and DES encrypted password for the username being auto-logged in.
#
# An example would be:
# itds:msu:cc./msu/35101d1a78a4060msul1:90090777f6d4956adba6590e878b0feb3aa8176180b303d3205c0faca2d5838
#
# A modified version of londes.js (called londes_auto.js) is required.
# This should be placed in $monitordir.
# Modifications include the following new lines at line 71.
# var uextkey = arguments[0];
# var lextkey = arguments[1];
# var upass = arguments[2];
# initkeys()
# var newpass = crypted(upass);
# print (newpass);
#
# and commenting out of lines (74 and 131)  which both refer to the window object:
# //     window.status="Initializing ...";
# //     window.status="Ready."
# as well as comming out the <script> and </script> tags at the beginning 
# and end of the file.
#
# The Rhino shell which allows Javascript to be called from the command line is # required. Download from http://www.mozilla.org/rhino/download.html 
# and put the js.jar file in $monitordir.  Then set the CLASSPATH in 
# .bash_profile of the user calling monitor.pl to include the path to js.jar
#
# CLASSPATH="/home/msuremot/monitoring/js.jar"
#                                                                            
# export PATH CLASSPATH
#
# The java run time envionment (i.e., a Java VM is needed).
#
# An additional file - recipients -should be added to a server's subdirectory   
# in $monitordir.  The file will contain an e-mail address of a user to
# receive notification when a particular failure condition is encountered on
# successive log-in attempts, and a list of failure types for which that 
# user should be notified.
#
# An example file would be:
# raeburn@msu.edu:conlost,unavailable,missingparam,unauthenticated,invalidcookie# ,nologin,uninitialized,rolesfailed,navmapfailed,logoutfailed
# itds@msu.edu:conlost
#
# In this case, raeburn@msu.edu would be contacted whenever consecutive failures
# of any of the failure types occurred; whereas itds@msu.edu would only be
# contacted if a conlost failure (no response from web server) occurred.
#
# The failure types are as follows:
# conlost  - Request for login page times out ( > 20s) or returns error
# unavailable - Server is unable to talk to itself via lonc/lond.
# missingparam - Log-in page returned too few parameters  -- 4 required   
#   usextkey, lextkey, serverid, logtoken
# unauthenticated - authentication failed -- afsdb0 inaccessible or /var full.
# invalidcookie - lonID cookie could not be extracted from HTTP header
# nologin - attempt to login resulted in error (401,403,404,500 etc.) 
# uninitialized - attempt to enter a course failed - could not initialize 
# rolesfailed - attempt to enter a course resulted in error (401,403,404,500) 
# navmapfailed - attempt to retrieve navmap failed - e.g., took  > 60 s, or 
# error (401,403,404,500 etc.)
# logoutfailed - attempt to logout after loading navmap failed.
#
# Page requests are submitted sequentially.  When a failure occurs, requests
# stop and actions are carried out for the specific failure type.
#
# If it is the second consecutive failure of this type, an e-mail will be
# sent to those recipients who are to be notified of this type of failure.
#
# If e-mail has already been sent for the current failure, no new message will 
# be sent.  Page loading times are determined (in ms) for the log-in page, 
# course initialization, and navmaps page. The plan is to store these load
# times and add to MRTG monitoring as a means of tracking server performance
# under different load conditions.
# 
# This script itself should be called with three parameters:
# encryption key (key used for DES encryption of the auto-user's password), 
# hostname of the server (e.g., s1.lite.ms.edu), 
# server nickname (e.g., alias).

my $monitordir = '/home/msuremot/monitoring';
my $contact_email = 'helpdesk@loncapa.org';

my $ua = LWP::UserAgent->new();
$ua->timeout(20);
my $loncookie_file = HTTP::Cookies->new();
my ($uname,$udom,$role,$upass);

if (@ARGV < 3) {
    print "Usage: $0 - encryptionkey hostname servernickname\n";
    exit(0);
}

my $keyphrase = $ARGV[0];
my $server = $ARGV[1];
my $serveralias = $ARGV[2];
my $authkeyfile = $monitordir.'/'.$serveralias.'/autouser.dat';

if (open (my $fh, "<$authkeyfile") ) {
    ($uname,$udom,$role,$upass) = split/:/,<$fh>;
    my $cipher = Crypt::CBC->new( {'key'     => $keyphrase,
                                   'cipher'  => 'DES'});
    $upass = $cipher->decrypt_hex($upass);
    close($fh);
}
my %recipients = ();
if (open (my $fh, "<$monitordir/$serveralias/recipients")) {
    my @buffer = <$fh>;
    close($fh);
    foreach my $line (@buffer) {
        my ($email,$conditions) = split/:/,$line;
        @{$recipients{$email}} = ();
        if ($conditions =~ m/,/) {
            @{$recipients{$email}} = split/,/,$conditions;
        } else {
            @{$recipients{$email}} = ("$conditions");
        }
    }
}

my @formitems = ('lextkey','uextkey','logtoken','serverid');
my %formvalues = ();
my %loadtimes = ();
my ($outcome,$loginpage,$lonid);
my @failures = ('conlost','unavailable','missingparam','unauthenticated','invalidcookie','nologin','unitialized','rolesfailed','navmapfailed','logoutfailed');

if (!-e "$monitordir/$serveralias") {
    mkdir("$monitordir/$serveralias",0755);
}
if (!-e "$monitordir/$serveralias/detected") {
    mkdir("$monitordir/$serveralias/detected",0755);
}
if (!-e "$monitordir/$serveralias/alert") {
    mkdir("$monitordir/$serveralias/alert",0755);
}

my $logfile = $monitordir.'/'.$serveralias.'/log'; 

$outcome = &attempt_access($outcome,$loginpage,$lonid,\@formitems,\%formvalues,\%loadtimes);
my $mailflag = &alertstatus($outcome,$server,$serveralias,\@failures,$logfile,$monitordir);
if ($mailflag) {
    my $mailresult = &mailalert($server,$outcome,$contact_email,\%recipients);
    open (my $logfh,">>$logfile");
    print $logfh localtime(time)." - sending mail - result = $mailresult\n";   
    close($logfh);
}

sub attempt_access {
    my ($outcome,$loginpage,$lonid,$formitems,$formvalues,$loadtimes) = @_;
    ($outcome,$loginpage) = &get_loginpage($server,$ua,$loadtimes);
    if ($outcome eq 'ok') {
        $ua->timeout(30);
        if ($loginpage) {
            $outcome = &parse_loginpage($loginpage,$formitems,$formvalues);
            if ($outcome eq 'ok') {
                ($outcome,$lonid) = &logmein($server,$uname,$udom,$upass,$ua,$loncookie_file,$formvalues,$loadtimes);
                if ($outcome eq 'ok')  {
                    if ($lonid) {
                        &setcookie($loncookie_file,$lonid,$server);
                        $outcome = &pickrole($server,$loncookie_file,$ua,$role,$loadtimes);
                        if ($outcome eq 'ok') {
                            $ua->timeout(120);
                            my $outcome = &loadnavmap($server,$loncookie_file,$ua,$loadtimes);
                            $ua->timeout(30);
                            if ($outcome eq 'ok') { 
                                $outcome = &logout($server,$loncookie_file,$ua);
                            } else {
                                &logout($server,$loncookie_file,$ua);
                            }
                            $loncookie_file->clear();
                        }
                    }
                }
            }
        }
    }
    return $outcome;
}

sub get_loginpage {
    my ($server,$ua) = @_;
    my ($outcome,$loginpage);
    my $URL = 'http://'.$server.'/adm/login';
    my $request = new HTTP::Request;
    $request =  GET $URL;
    my $res = $ua->request($request);
    if ($res->is_success) {
        my $dump = $res->content;
        $outcome = 'ok';
        $loginpage = $dump;
    } else {
        $outcome = 'conlost';
    }
    return ($outcome,$loginpage);
}

sub parse_loginpage {
    my ($loginpage,$formitems,$formvalues) = @_;
    my $outcome = 0;
    my $paramcount = 0;
    if ($loginpage =~ m-src="/adm/lonKaputt/lonlogo_broken\.gif"-) {
        $outcome = "unavailable";
    } else {
        foreach (@{$formitems}) {
            if ($loginpage =~ m/<input\stype="hidden"\sname="$_"\svalue="(-?\w+)"\s?\/?>/) {
                $$formvalues{$_} = $1;
                $paramcount ++;
            }
        }
        if ($paramcount = scalar(@formitems)) {
            $outcome = 'ok';
        } else {
            $outcome = 'missingparam';
        }
    }
    return $outcome;
}
sub logmein {
    my ($server,$uname,$udom,$upass,$ua,$loncookie_file,$formvalues) = @_;
    my ($outcome,$lonid);
    open(PIPE,"-|") || exec "java org.mozilla.javascript.tools.shell.Main londes_auto.js $$formvalues{uextkey} $$formvalues{lextkey} $upass";
    my $cryppass = <PIPE>;
    close PIPE;
    my $URL = 'http://'.$server.'/adm/authenticate';
    $ua->cookie_jar( $loncookie_file );
    my $req = POST $URL,
      Content_Type => 'application/x-www-form-urlencoded',
      Content      =>
      [
        uname => "$uname",
        udom => "$udom",
        upass => "$cryppass",
        logtoken => "$$formvalues{logtoken}",
        serverid => "$$formvalues{serverid}",
      ];
    my $start = [gettimeofday];
    my $res = $ua->request($req);
    if ($res->is_success) {
        my $dump = $res->content;
        if ($dump =~ m-Username\sand\\or\spassword\scould\snot\sbe\sauthenticated-) {
            $outcome = 'unauthenticated';
        }
        $loncookie_file->extract_cookies($res);
        my $cookie = $loncookie_file->as_string;
        if ($cookie =~ m/lonID=(\w+);/) {
            $lonid = $1;
            $outcome = 'ok';
        } else {
            $outcome = 'invalidcookie';
        }
        my $end = [gettimeofday];
        $loadtimes{login} = tv_interval $start, $end;
        $loadtimes{login} *= 1000;
    } else {
        $outcome = 'nologin';
    }
    return ($outcome,$lonid);
}

sub pickrole {
    my ($server,$loncookie_file,$ua,$role) = @_;
    my ($start,$end,$outcome);
    my $URL = 'http://'.$server.'/adm/roles';
    my $request = POST $URL,
      Content      =>
      [
        selectrole => '1',
        $role => "Select",
      ];
    $loncookie_file->add_cookie_header($request);
    my $response = $ua->request($request);
    $start = [gettimeofday];
    if ($response->is_success) {
        my $dump = $response->content;
        if ($dump =~ m-<a href="/adm/roles?tryagain=1">-) {
            print "initializing course failed\n";
            $outcome = 'uninitialized';
        } else {
            $outcome = 'ok';
        }
        $end = [gettimeofday];
        $loadtimes{initialize} = tv_interval $start, $end;
        $loadtimes{initialize} *= 1000; 
    } else {
        $outcome = 'rolesfailed';
    }
    return $outcome;
}

sub loadnavmap {
    my ($server,$loncookie_file,$ua,$loadtimes) = @_;
    my $outcome;
    my $URL = 'http://'.$server.'/adm/navmaps';
    my $request = new HTTP::Request;
    $request =  GET $URL;
    my $start = [gettimeofday];
    my $res = $ua->request($request);
    if ($res->is_success) {
        my $dump = $res->content;
        my $end = [gettimeofday];
        $$loadtimes{navmap} = tv_interval $start, $end;
        $$loadtimes{navmap} *= 1000;
        $outcome = 'ok';
    } else {
        $outcome = 'navmapfailed';
    }
    return $outcome;
}

sub logout {
    my ($server,$loncookie_file,$ua) = @_;
    my $outcome;
    my $URL = 'http://'.$server.'/adm/logout';
    my $request = new HTTP::Request;
    $request =  GET $URL;
    my $res = $ua->request($request);
    if ($res->is_success) {
        my $dump = $res->content;
        $outcome = 'ok';
    } else {
        $outcome = 'logoutfailed';
    }
    return $outcome;
}

sub setcookie {
    my ($loncookie_file,$lonid,$server) = @_;
    $loncookie_file->set_cookie(0,"lonID",$lonid,"/","$server",undef,0,0,60,0);
}

sub alertstatus {
    my ($outcome,$server,$serveralias,$failures,$logfile,$monitordir) = @_;
    my $counter = 0;
    my $mailflag = 0;
    open(my $logfh,">>$logfile");
    while ($counter<@{$failures}) {
        if ($outcome eq $failures[$counter]) {
            if (-e "$monitordir/$serveralias/detected/$$failures[$counter]") {
                if  (-e "$monitordir/$serveralias/alert/$$failures[$counter]") {
                    print $logfh localtime(time)." - $outcome - current alert\n";
                } else {
                    $mailflag = 1;
                    if (open(my $fh,">$monitordir/$serveralias/alert/$$failures[$counter]")) {
                        print $fh time;
                        close $fh;
                        print $logfh localtime(time)." - $outcome - new alert sent\n";
                    }
                }
            } else {
                if (open(my $fh,">$monitordir/$serveralias/detected/$$failures[$counter]")) {
                    print $fh time;
                    close $fh;
                    print $logfh localtime(time)." - $outcome - first failure\n";
                }
            }
        } else {
            if (-e "$monitordir/$serveralias/detected/$$failures[$counter]") {
                unlink("$monitordir/$serveralias/detected/$$failures[$counter]");
                print $logfh localtime(time)." - $$failures[$counter] - old failure cleared\n";

            }
            if (-e "$monitordir/$serveralias/alert/$$failures[$counter]") {
                unlink("$monitordir/$serveralias/alert/$$failures[$counter]");
                print $logfh localtime(time)." - $$failures[$counter] - old alert cleared\n";
            }
        }
        $counter ++;
    }
    close($logfh);
    return $mailflag;
}

sub mailalert {
    my ($server,$failure,$contact_email,$recipients) = @_;
    my $outcome;
    my $toline = "To: ";
    foreach my $email (keys %{$recipients}) {
        if (grep/^$failure$/,@{$$recipients{$email}}) {
            $toline .= $email.',';
        }
    }
    $toline =~ s/,$//;
    my $alertmail = "$toline\n".
                    "From: $contact_email\n".
                    "Subject: MSU LON-CAPA monitoring\n".
                    "\n\nTwo successive failures of type: $failure on $server\n\n";
    if (open(MAIL, "|/usr/lib/sendmail -oi -t -odb")) {
        print MAIL $alertmail;
        close(MAIL);
        $outcome = 'ok';
    } else {
        $outcome = 'fail';
    }
    return $outcome;
}

--raeburn1112049539--