[LON-CAPA-cvs] cvs: loncom /metadata_database searchcat.pl /metadata_database/LONCAPA lonmetadata.pm
matthew
lon-capa-cvs@mail.lon-capa.org
Mon, 12 Apr 2004 21:11:45 -0000
This is a MIME encoded message
--matthew1081804305
Content-Type: text/plain
matthew Mon Apr 12 17:11:45 2004 EDT
Modified files:
/loncom/metadata_database searchcat.pl
/loncom/metadata_database/LONCAPA lonmetadata.pm
Log:
Moved nohist_reseval parsing functionality from searchcat.pl to lonmetadata.pm.
searchcat.pl: stop dying on insert errors.
lonmetadata.pm: &store_metadata now translates 'nan' to 'NULL' because MySQL
won't take 'nan' in a numeric field.
Added &process_reseval_data and &process_dynamic_metadata routines.
--matthew1081804305
Content-Type: text/plain
Content-Disposition: attachment; filename="matthew-20040412171145.txt"
Index: loncom/metadata_database/searchcat.pl
diff -u loncom/metadata_database/searchcat.pl:1.56 loncom/metadata_database/searchcat.pl:1.57
--- loncom/metadata_database/searchcat.pl:1.56 Fri Apr 9 18:04:53 2004
+++ loncom/metadata_database/searchcat.pl Mon Apr 12 17:11:45 2004
@@ -2,7 +2,7 @@
# The LearningOnline Network
# searchcat.pl "Search Catalog" batch script
#
-# $Id: searchcat.pl,v 1.56 2004/04/09 22:04:53 matthew Exp $
+# $Id: searchcat.pl,v 1.57 2004/04/12 21:11:45 matthew Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -144,6 +144,8 @@
# Let people know we are running
open(LOG,'>'.$perlvar{'lonDaemons'}.'/logs/searchcat.log');
&log(0,'==== Searchcat Run '.localtime()."====");
+
+
if ($debug) {
&log(0,'simulating') if ($simulate);
&log(0,'only processing user '.$oneuser) if ($oneuser);
@@ -305,7 +307,6 @@
$_=$file;
}
-
##
## process_meta_file
## Called by File::Find.
@@ -358,11 +359,9 @@
\%Data);
if ($err) {
&log(0,"MySQL Error Insert: ".$err);
- die $err;
}
if ($count < 1) {
&log(0,"Unable to insert record into MySQL database for $url");
- die "Unable to insert record into MySQl database for $url";
}
}
#
@@ -491,33 +490,7 @@
return 0;
}
#
- # Process every stored element
- while (my ($storedkey,$value) = each(%evaldata)) {
- my ($source,$file,$type) = split('___',$storedkey);
- $source = &unescape($source);
- $file = &unescape($file);
- $value = &unescape($value);
- " got ".$file."\n ".$type." ".$source."\n";
- if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) {
- #
- # Statistics: $source is course id
- $DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value;
- } elsif ($type =~ /^(clear|comments|depth|technical|helpful)$/){
- #
- # Evaluation $source is username, check if they evaluated it
- # more than once. If so, pad the entry with a space.
- while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) {
- $source .= ' ';
- }
- $DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value;
- } elsif ($type =~ /^(course|comefrom|goto|usage)$/) {
- #
- # Context $source is course id or resource
- push(@{$DynamicData{$file}->{$type}},&unescape($source));
- } else {
- &log(0," ".$user."@".$dom.":Process metadata: Unable to decode ".$type);
- }
- }
+ %DynamicData = &LONCAPA::lonmetadata::process_reseval_data(\%evaldata);
untie(%evaldata);
#
# Read in the access count data
@@ -551,71 +524,17 @@
&log(7,' No dynamic data for '.$url) if ($debug);
return ();
}
- my %data;
- my $resdata = $DynamicData{$url};
- #
- # Get the statistical data
- foreach my $type (qw/avetries difficulty stdno/) {
- my $count;
- my $sum;
- my @Values;
- foreach my $coursedata (values(%{$resdata->{'statistics'}})) {
- if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
- $count++;
- $sum += $coursedata->{$type};
- push(@Values,$coursedata->{$type});
- }
- }
- if ($count) {
- $data{$type} = $sum/$count;
- $data{$type.'_list'} = join(',',@Values);
- }
- }
+ my %data = &LONCAPA::lonmetadata::process_dynamic_metadata($url,
+ \%DynamicData);
# find the count
$data{'count'} = $Counts{$url};
#
- # Get the context data
- foreach my $type (qw/course goto comefrom/) {
- if (defined($resdata->{$type}) &&
- ref($resdata->{$type}) eq 'ARRAY') {
- $data{$type} = scalar(@{$resdata->{$type}});
- $data{$type.'_list'} = join(',',@{$resdata->{$type}});
- }
- }
- if (defined($resdata->{'usage'}) &&
- ref($resdata->{'usage'}) eq 'ARRAY') {
- $data{'sequsage'} = scalar(@{$resdata->{'usage'}});
- $data{'sequsage_list'} = join(',',@{$resdata->{'usage'}});
- }
- #
- # Get the evaluation data
- foreach my $type (qw/clear technical correct helpful depth/) {
- my $count;
- my $sum;
- foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){
- $sum += $resdata->{'evaluation'}->{$type}->{$evaluator};
- $count++;
- }
- if ($count > 0) {
- $data{$type}=$sum/$count;
- }
- }
- #
- # put together comments
- my $comments = '<div class="LCevalcomments">';
- foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){
- $comments .= $evaluator.':'.
- $resdata->{'evaluation'}->{'comments'}->{$evaluator}.'<hr />';
- }
- $comments .= '</div>';
- #
# Log the dynamic metadata
if ($debug) {
while (my($k,$v)=each(%data)) {
&log(8," ".$k." => ".$v);
}
}
- #
return %data;
}
Index: loncom/metadata_database/LONCAPA/lonmetadata.pm
diff -u loncom/metadata_database/LONCAPA/lonmetadata.pm:1.4 loncom/metadata_database/LONCAPA/lonmetadata.pm:1.5
--- loncom/metadata_database/LONCAPA/lonmetadata.pm:1.4 Thu Apr 8 10:50:44 2004
+++ loncom/metadata_database/LONCAPA/lonmetadata.pm Mon Apr 12 17:11:45 2004
@@ -1,6 +1,6 @@
# The LearningOnline Network with CAPA
#
-# $Id: lonmetadata.pm,v 1.4 2004/04/08 14:50:44 matthew Exp $
+# $Id: lonmetadata.pm,v 1.5 2004/04/12 21:11:45 matthew Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -297,7 +297,11 @@
my @MData;
foreach my $field (@Metadata_Table_Description) {
if (exists($mdata->{$field->{'name'}})) {
- push(@MData,$mdata->{$field->{'name'}});
+ if ($mdata->{$field->{'name'}} eq 'nan') {
+ push(@MData,'NULL');
+ } else {
+ push(@MData,$mdata->{$field->{'name'}});
+ }
} else {
push(@MData,undef);
}
@@ -378,6 +382,203 @@
######################################################################
######################################################################
+
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &process_reseval_data
+
+Process a nohist_resevaldata hash into a more complex data structure.
+
+Input: Hash reference containing reseval data
+
+Returns: Hash with the following structure:
+
+$hash{$url}->{'statistics'}->{$courseid}->{'avetries'} = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'count'} = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'difficulty'} = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'stdno'} = $value
+$hash{$url}->{'statistics'}->{$courseid}->{'timestamp'} = $value
+
+$hash{$url}->{'evaluation'}->{$username}->{'clear'} = $value
+$hash{$url}->{'evaluation'}->{$username}->{'comments'} = $value
+$hash{$url}->{'evaluation'}->{$username}->{'depth'} = $value
+$hash{$url}->{'evaluation'}->{$username}->{'technical'} = $value
+$hash{$url}->{'evaluation'}->{$username}->{'helpful'} = $value
+
+$hash{$url}->{'course'} = \@Courses
+$hash{$url}->{'comefrom'} = \@Resources
+$hash{$url}->{'goto'} = \@Resources
+$hash{$url}->{'usage'} = \@Resources
+
+$hash{$url}->{'stats'}->{$courseid\_$section}->{$key} = $value
+
+=cut
+
+######################################################################
+######################################################################
+sub process_reseval_data {
+ my ($evaldata) = @_;
+ my %DynamicData;
+ #
+ # Process every stored element
+ while (my ($storedkey,$value) = each(%{$evaldata})) {
+ my ($source,$file,$type) = split('___',$storedkey);
+ $source = &unescape($source);
+ $file = &unescape($file);
+ $value = &unescape($value);
+ " got ".$file."\n ".$type." ".$source."\n";
+ if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) {
+ #
+ # Statistics: $source is course id
+ $DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value;
+ } elsif ($type =~ /^(clear|comments|depth|technical|helpful)$/){
+ #
+ # Evaluation $source is username, check if they evaluated it
+ # more than once. If so, pad the entry with a space.
+ while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) {
+ $source .= ' ';
+ }
+ $DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value;
+ } elsif ($type =~ /^(course|comefrom|goto|usage)$/) {
+ #
+ # Context $source is course id or resource
+ push(@{$DynamicData{$file}->{$type}},&unescape($source));
+ } elsif ($type eq 'stats') {
+ #
+ # Statistics storage...
+ # $source is $cid\_$sec\_$stdno
+ # $value is stat1=value&stat2=value&stat3=value,....
+ #
+ my ($cid,$sec,$stdno)=split('_',$source);
+ my $crssec = $cid.'_'.$sec;
+ my @Data = split('&',$value);
+ my %Statistics;
+ while (my ($key,$value) = split('=',pop(@Data))) {
+ $Statistics{$key} = $value;
+ }
+ #
+ # Only store the data if the number of students is greater
+ # than the data already stored
+ if (! exists($DynamicData{$file}->{'stats'}->{$crssec}) ||
+ $DynamicData{$file}->{'stats'}->{$crssec}->{'stdno'}<$stdno){
+ $DynamicData{$file}->{'stats'}->{$crssec}=\%Statistics;
+ }
+ }
+ }
+ return %DynamicData;
+}
+
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &process_dynamic_metadata
+
+Inputs: $url: the url of the item to process
+$DynamicData: hash reference for the results of &process_reseval_data
+
+Returns: Hash containing the following keys:
+ avetries, avetries_list, difficulty, difficulty_list, stdno, stdno_list,
+ course, course_list, goto, goto_list, comefrom, comefrom_list,
+ usage, clear, technical, correct, helpful, depth, comments
+
+ Each of the return keys is associated with either a number or a string
+ The *_list items are comma-seperated strings. 'comments' is a string
+ containing generically marked-up comments.
+
+=cut
+
+######################################################################
+######################################################################
+sub process_dynamic_metadata {
+ my ($url,$DynamicData) = @_;
+ my %data;
+ my $resdata = $DynamicData->{$url};
+ #
+ # Get the statistical data
+ foreach my $type (qw/avetries difficulty stdno/) {
+ my $count;
+ my $sum;
+ my @Values;
+ #
+ foreach my $coursedata (values(%{$resdata->{'statistics'}}),
+ values(%{$resdata->{'stats'}})) {
+ if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
+ $count++;
+ $sum += $coursedata->{$type};
+ push(@Values,$coursedata->{$type});
+ }
+ }
+ if ($count) {
+ $data{$type} = $sum/$count;
+ $data{$type.'_list'} = join(',',@Values);
+ }
+ }
+ #
+ # Get the context data
+ foreach my $type (qw/course goto comefrom/) {
+ if (defined($resdata->{$type}) &&
+ ref($resdata->{$type}) eq 'ARRAY') {
+ $data{$type} = scalar(@{$resdata->{$type}});
+ $data{$type.'_list'} = join(',',@{$resdata->{$type}});
+ }
+ }
+ if (defined($resdata->{'usage'}) &&
+ ref($resdata->{'usage'}) eq 'ARRAY') {
+ $data{'sequsage'} = scalar(@{$resdata->{'usage'}});
+ $data{'sequsage_list'} = join(',',@{$resdata->{'usage'}});
+ }
+ #
+ # Get the evaluation data
+ foreach my $type (qw/clear technical correct helpful depth/) {
+ my $count;
+ my $sum;
+ foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){
+ $sum += $resdata->{'evaluation'}->{$type}->{$evaluator};
+ $count++;
+ }
+ if ($count > 0) {
+ $data{$type}=$sum/$count;
+ }
+ }
+ #
+ # put together comments
+ my $comments = '<div class="LCevalcomments">';
+ foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){
+ $comments .= '<span class="author">'.$evaluator.'</span>'.
+ '<span class="comment">'.
+ $resdata->{'evaluation'}->{'comments'}->{$evaluator}.'</span>';
+ }
+ $comments .= '</div>';
+ #
+ return %data;
+}
+
+######################################################################
+######################################################################
+##
+## The usual suspects, repeated here to reduce dependency hell
+##
+######################################################################
+######################################################################
+sub unescape {
+ my $str=shift;
+ $str =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
+ return $str;
+}
+
+sub escape {
+ my $str=shift;
+ $str =~ s/(\W)/"%".unpack('H2',$1)/eg;
+ return $str;
+}
+
1;
--matthew1081804305--