[LON-CAPA-cvs] cvs: loncom /metadata_database parse_activity_log.pl

[matthew]

matthew		Tue Sep 20 13:49:53 2005 EDT

  Modified files:              
    /loncom/metadata_database	parse_activity_log.pl 
  Log:
  Better fix for last commit - if a $chunk is over 20k in length, store it
  in a file, "activity.log.unprocessed_chunks", and skip inserting it into the
  database.
  
  
Index: loncom/metadata_database/parse_activity_log.pl
diff -u loncom/metadata_database/parse_activity_log.pl:1.18 loncom/metadata_database/parse_activity_log.pl:1.19
--- loncom/metadata_database/parse_activity_log.pl:1.18	Tue Sep 20 12:50:40 2005
+++ loncom/metadata_database/parse_activity_log.pl	Tue Sep 20 13:49:51 2005
@@ -2,7 +2,7 @@
 #
 # The LearningOnline Network
 #
-# $Id: parse_activity_log.pl,v 1.18 2005/09/20 16:50:40 matthew Exp $
+# $Id: parse_activity_log.pl,v 1.19 2005/09/20 17:49:51 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -144,6 +144,7 @@
 my $sourcefilename;   # activity log data
 my $newfilename;      # $sourcefilename will be renamed to this
 my $error_filename;   # Errors in parsing the activity log will be written here
+my $chunk_filename;   # where we save data we are not going to write to db
 if ($srcfile) {
     $sourcefilename = $srcfile;
 } else {
@@ -153,6 +154,8 @@
 $sql_filename =~ s|[^/]*$|activity.log.sql|;
 my $gz_sql_filename = $sql_filename.'.gz';
 #
+$chunk_filename = $sourcefilename.".unprocessed_chunks";
+#
 my $xml_filename = $sourcefilename;
 my $gz_xml_filename = $xml_filename.'.gz';
 if (defined($xmlfile)) {
@@ -486,20 +489,22 @@
         if (! defined($host)) { $host = 'unknown'; }
         my $prevchunk = 'none';
         foreach my $chunk (split(/\&\&\&/,$log)) {
-            my $warningflag = '';
-	    my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk);
-            # 
-            # Sometimes we get a file pasted into the activity.log from
-            # an upload form.  Here we try to detect it and avoid inserting
-            # it into the database to avoid the quiet death of the database
-            # connection
-            my $i;
-            for ($i=0;$i<$#values;$i++) {
-                if ($values[$i] =~ /^HWVAL/) {
-                    $#values = $i;
-                    last;
+            if (length($chunk) > 20000) {
+                # avoid putting too much data into the database
+                # (usually an uploaded file or something similar)
+                $chunk = &escape($chunk);
+                if (! open(CHUNKFILE,">>$chunk_filename") ||
+                    ! print CHUNKFILE $timestamp.':'.$host.':'.$chunk.$/) {
+                    # abort
+                    close(CHUNKFILE);
+                    close(IN);
+                    return undef;
                 }
+                close(CHUNKFILE);
+                next;
             }
+            my $warningflag = '';
+	    my ($time,$res,$uname,$udom,$action,@values)= split(/:/,$chunk);
             #
             if (! defined($res) || $res =~ /^\s*$/) {
                 $res = '/adm/roles';