[LON-CAPA-cvs] cvs: loncom /metadata_database searchcat.pl /metadata_database/LONCAPA lonmetadata.pm

matthew lon-capa-cvs@mail.lon-capa.org
Mon, 12 Apr 2004 21:11:45 -0000


This is a MIME encoded message

--matthew1081804305
Content-Type: text/plain

matthew		Mon Apr 12 17:11:45 2004 EDT

  Modified files:              
    /loncom/metadata_database	searchcat.pl 
    /loncom/metadata_database/LONCAPA	lonmetadata.pm 
  Log:
  Moved nohist_reseval parsing functionality from searchcat.pl to lonmetadata.pm.
  searchcat.pl: stop dying on insert errors.
  lonmetadata.pm: &store_metadata now translates 'nan' to 'NULL' because MySQL
  won't take 'nan' in a numeric field.
  Added &process_reseval_data and &process_dynamic_metadata routines.
  
  
--matthew1081804305
Content-Type: text/plain
Content-Disposition: attachment; filename="matthew-20040412171145.txt"

Index: loncom/metadata_database/searchcat.pl
diff -u loncom/metadata_database/searchcat.pl:1.56 loncom/metadata_database/searchcat.pl:1.57
--- loncom/metadata_database/searchcat.pl:1.56	Fri Apr  9 18:04:53 2004
+++ loncom/metadata_database/searchcat.pl	Mon Apr 12 17:11:45 2004
@@ -2,7 +2,7 @@
 # The LearningOnline Network
 # searchcat.pl "Search Catalog" batch script
 #
-# $Id: searchcat.pl,v 1.56 2004/04/09 22:04:53 matthew Exp $
+# $Id: searchcat.pl,v 1.57 2004/04/12 21:11:45 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -144,6 +144,8 @@
 # Let people know we are running
 open(LOG,'>'.$perlvar{'lonDaemons'}.'/logs/searchcat.log');
 &log(0,'==== Searchcat Run '.localtime()."====");
+
+
 if ($debug) {
     &log(0,'simulating') if ($simulate);
     &log(0,'only processing user '.$oneuser) if ($oneuser);
@@ -305,7 +307,6 @@
     $_=$file;
 }
 
-
 ##
 ## process_meta_file
 ##   Called by File::Find. 
@@ -358,11 +359,9 @@
                                                                  \%Data);
         if ($err) {
             &log(0,"MySQL Error Insert: ".$err);
-            die $err;
         }
         if ($count < 1) {
             &log(0,"Unable to insert record into MySQL database for $url");
-            die "Unable to insert record into MySQl database for $url";
         }
     }
     #
@@ -491,33 +490,7 @@
         return 0;
     }
     #
-    # Process every stored element
-    while (my ($storedkey,$value) = each(%evaldata)) {
-        my ($source,$file,$type) = split('___',$storedkey);
-        $source = &unescape($source);
-        $file = &unescape($file);
-        $value = &unescape($value);
-         "    got ".$file."\n        ".$type." ".$source."\n";
-        if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) {
-            #
-            # Statistics: $source is course id
-            $DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value;
-        } elsif ($type =~ /^(clear|comments|depth|technical|helpful)$/){
-            #
-            # Evaluation $source is username, check if they evaluated it
-            # more than once.  If so, pad the entry with a space.
-            while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) {
-                $source .= ' ';
-            }
-            $DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value;
-        } elsif ($type =~ /^(course|comefrom|goto|usage)$/) {
-            #
-            # Context $source is course id or resource
-            push(@{$DynamicData{$file}->{$type}},&unescape($source));
-        } else {
-            &log(0,"   ".$user."@".$dom.":Process metadata: Unable to decode ".$type);
-        }
-    }
+    %DynamicData = &LONCAPA::lonmetadata::process_reseval_data(\%evaldata);
     untie(%evaldata);
     #
     # Read in the access count data
@@ -551,71 +524,17 @@
         &log(7,'    No dynamic data for '.$url) if ($debug);
         return ();
     }
-    my %data;
-    my $resdata = $DynamicData{$url};
-    #
-    # Get the statistical data
-    foreach my $type (qw/avetries difficulty stdno/) {
-        my $count;
-        my $sum;
-        my @Values;
-        foreach my $coursedata (values(%{$resdata->{'statistics'}})) {
-            if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
-                $count++;
-                $sum += $coursedata->{$type};
-                push(@Values,$coursedata->{$type});
-            }
-        }
-        if ($count) {
-            $data{$type} = $sum/$count;
-            $data{$type.'_list'} = join(',',@Values);
-        }
-    }
+    my %data = &LONCAPA::lonmetadata::process_dynamic_metadata($url,
+                                                               \%DynamicData);
     # find the count
     $data{'count'} = $Counts{$url};
     #
-    # Get the context data
-    foreach my $type (qw/course goto comefrom/) {
-        if (defined($resdata->{$type}) && 
-            ref($resdata->{$type}) eq 'ARRAY') {
-            $data{$type} = scalar(@{$resdata->{$type}});
-            $data{$type.'_list'} = join(',',@{$resdata->{$type}});
-        }
-    }
-    if (defined($resdata->{'usage'}) && 
-        ref($resdata->{'usage'}) eq 'ARRAY') {
-        $data{'sequsage'} = scalar(@{$resdata->{'usage'}});
-        $data{'sequsage_list'} = join(',',@{$resdata->{'usage'}});
-    }
-    #
-    # Get the evaluation data
-    foreach my $type (qw/clear technical correct helpful depth/) {
-        my $count;
-        my $sum;
-        foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){
-            $sum += $resdata->{'evaluation'}->{$type}->{$evaluator};
-            $count++;
-        }
-        if ($count > 0) {
-            $data{$type}=$sum/$count;
-        }
-    }
-    #
-    # put together comments
-    my $comments = '<div class="LCevalcomments">';
-    foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){
-        $comments .= $evaluator.':'.
-            $resdata->{'evaluation'}->{'comments'}->{$evaluator}.'<hr />';
-    }
-    $comments .= '</div>';
-    #
     # Log the dynamic metadata
     if ($debug) {
         while (my($k,$v)=each(%data)) {
             &log(8,"    ".$k." => ".$v);
         }
     }
-    #
     return %data;
 }
 
Index: loncom/metadata_database/LONCAPA/lonmetadata.pm
diff -u loncom/metadata_database/LONCAPA/lonmetadata.pm:1.4 loncom/metadata_database/LONCAPA/lonmetadata.pm:1.5
--- loncom/metadata_database/LONCAPA/lonmetadata.pm:1.4	Thu Apr  8 10:50:44 2004
+++ loncom/metadata_database/LONCAPA/lonmetadata.pm	Mon Apr 12 17:11:45 2004
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonmetadata.pm,v 1.4 2004/04/08 14:50:44 matthew Exp $
+# $Id: lonmetadata.pm,v 1.5 2004/04/12 21:11:45 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -297,7 +297,11 @@
         my @MData;
         foreach my $field (@Metadata_Table_Description) {
             if (exists($mdata->{$field->{'name'}})) {
-                push(@MData,$mdata->{$field->{'name'}});
+                if ($mdata->{$field->{'name'}} eq 'nan') {
+                    push(@MData,'NULL');
+                } else {
+                    push(@MData,$mdata->{$field->{'name'}});
+                }
             } else {
                 push(@MData,undef);
             }
@@ -378,6 +382,203 @@
 
 ######################################################################
 ######################################################################
+
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &process_reseval_data 
+
+Process a nohist_resevaldata hash into a more complex data structure.
+
+Input: Hash reference containing reseval data
+
+Returns: Hash with the following structure:
+
+$hash{$url}->{'statistics'}->{$courseid}->{'avetries'}   = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'count'}      = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'difficulty'} = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'stdno'}      = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'timestamp'}  = $value
+
+$hash{$url}->{'evaluation'}->{$username}->{'clear'}     = $value
+$hash{$url}->{'evaluation'}->{$username}->{'comments'}  = $value
+$hash{$url}->{'evaluation'}->{$username}->{'depth'}     = $value
+$hash{$url}->{'evaluation'}->{$username}->{'technical'} = $value
+$hash{$url}->{'evaluation'}->{$username}->{'helpful'}   = $value
+
+$hash{$url}->{'course'}    = \@Courses
+$hash{$url}->{'comefrom'}  = \@Resources
+$hash{$url}->{'goto'}      = \@Resources
+$hash{$url}->{'usage'}     = \@Resources
+
+$hash{$url}->{'stats'}->{$courseid\_$section}->{$key} = $value
+
+=cut
+
+######################################################################
+######################################################################
+sub process_reseval_data {
+    my ($evaldata) = @_;
+    my %DynamicData;
+    #
+    # Process every stored element
+    while (my ($storedkey,$value) = each(%{$evaldata})) {
+        my ($source,$file,$type) = split('___',$storedkey);
+        $source = &unescape($source);
+        $file = &unescape($file);
+        $value = &unescape($value);
+         "    got ".$file."\n        ".$type." ".$source."\n";
+        if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) {
+            #
+            # Statistics: $source is course id
+            $DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value;
+        } elsif ($type =~ /^(clear|comments|depth|technical|helpful)$/){
+            #
+            # Evaluation $source is username, check if they evaluated it
+            # more than once.  If so, pad the entry with a space.
+            while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) {
+                $source .= ' ';
+            }
+            $DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value;
+        } elsif ($type =~ /^(course|comefrom|goto|usage)$/) {
+            #
+            # Context $source is course id or resource
+            push(@{$DynamicData{$file}->{$type}},&unescape($source));
+        } elsif ($type eq 'stats') {
+            #
+            # Statistics storage...
+            # $source is $cid\_$sec\_$stdno
+            # $value is stat1=value&stat2=value&stat3=value,....
+            #
+            my ($cid,$sec,$stdno)=split('_',$source);
+            my $crssec = $cid.'_'.$sec;
+            my @Data = split('&',$value);
+            my %Statistics;
+            while (my ($key,$value) = split('=',pop(@Data))) {
+                $Statistics{$key} = $value;
+            }
+            #
+            # Only store the data if the number of students is greater
+            # than the data already stored
+            if (! exists($DynamicData{$file}->{'stats'}->{$crssec}) ||
+                $DynamicData{$file}->{'stats'}->{$crssec}->{'stdno'}<$stdno){
+                $DynamicData{$file}->{'stats'}->{$crssec}=\%Statistics;
+            }
+        }
+    }
+    return %DynamicData;
+}
+
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &process_dynamic_metadata
+
+Inputs: $url: the url of the item to process
+$DynamicData: hash reference for the results of &process_reseval_data
+
+Returns: Hash containing the following keys:
+    avetries, avetries_list, difficulty, difficulty_list, stdno, stdno_list,
+    course, course_list, goto, goto_list, comefrom, comefrom_list,
+    usage, clear, technical, correct, helpful, depth, comments
+
+    Each of the return keys is associated with either a number or a string
+    The *_list items are comma-seperated strings.  'comments' is a string
+    containing generically marked-up comments.
+
+=cut
+
+######################################################################
+######################################################################
+sub process_dynamic_metadata {
+    my ($url,$DynamicData) = @_;
+    my %data;
+    my $resdata = $DynamicData->{$url};
+    #
+    # Get the statistical data
+    foreach my $type (qw/avetries difficulty stdno/) {
+        my $count;
+        my $sum;
+        my @Values;
+        #
+        foreach my $coursedata (values(%{$resdata->{'statistics'}}),
+                                values(%{$resdata->{'stats'}})) {
+            if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
+                $count++;
+                $sum += $coursedata->{$type};
+                push(@Values,$coursedata->{$type});
+            }
+        }
+        if ($count) {
+            $data{$type} = $sum/$count;
+            $data{$type.'_list'} = join(',',@Values);
+        }
+    }
+    #
+    # Get the context data
+    foreach my $type (qw/course goto comefrom/) {
+        if (defined($resdata->{$type}) && 
+            ref($resdata->{$type}) eq 'ARRAY') {
+            $data{$type} = scalar(@{$resdata->{$type}});
+            $data{$type.'_list'} = join(',',@{$resdata->{$type}});
+        }
+    }
+    if (defined($resdata->{'usage'}) && 
+        ref($resdata->{'usage'}) eq 'ARRAY') {
+        $data{'sequsage'} = scalar(@{$resdata->{'usage'}});
+        $data{'sequsage_list'} = join(',',@{$resdata->{'usage'}});
+    }
+    #
+    # Get the evaluation data
+    foreach my $type (qw/clear technical correct helpful depth/) {
+        my $count;
+        my $sum;
+        foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){
+            $sum += $resdata->{'evaluation'}->{$type}->{$evaluator};
+            $count++;
+        }
+        if ($count > 0) {
+            $data{$type}=$sum/$count;
+        }
+    }
+    #
+    # put together comments
+    my $comments = '<div class="LCevalcomments">';
+    foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){
+        $comments .= '<span class="author">'.$evaluator.'</span>'.
+            '<span class="comment">'.
+            $resdata->{'evaluation'}->{'comments'}->{$evaluator}.'</span>';
+    }
+    $comments .= '</div>';
+    #
+    return %data;
+}
+
+######################################################################
+######################################################################
+##
+## The usual suspects, repeated here to reduce dependency hell
+##
+######################################################################
+######################################################################
+sub unescape {
+    my $str=shift;
+    $str =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
+    return $str;
+}
+
+sub escape {
+    my $str=shift;
+    $str =~ s/(\W)/"%".unpack('H2',$1)/eg;
+    return $str;
+}
+
 
 1;
 

--matthew1081804305--