[LON-CAPA-cvs] cvs: loncom /interface loncoursedata.pm /interface/statistics lonproblemanalysis.pm

matthew lon-capa-cvs@mail.lon-capa.org
Mon, 20 Oct 2003 20:42:39 -0000


This is a MIME encoded message

--matthew1066682559
Content-Type: text/plain

matthew		Mon Oct 20 16:42:39 2003 EDT

  Modified files:              
    /loncom/interface	loncoursedata.pm 
    /loncom/interface/statistics	lonproblemanalysis.pm 
  Log:
  Excel output for optionresponse analysis.
  
  loncoursedata.pm: 
  Added &get_student_data.
  Modified return values of get_optionresionse_data it now logs errors and
  (on error) no longer returns a reference to useless data.
  
  lonproblemanalysis.pm:
  Added 'Produce Excel Data Sheet' button.  Wording will probably change, as
  will page which presents the spreadsheet for download.
  Added a bunch of routines associated with excel output, including two
  routines 'calc_serial' and '_adjustment' copied from Andrew Benham's
  datecalc1.pl, part of the Spreadsheet::WriteExcel CPAN module (GPL).
  Read through some of the comments on these to learn a little about date
  handling in Excel.
  
  
--matthew1066682559
Content-Type: text/plain
Content-Disposition: attachment; filename="matthew-20031020164239.txt"

Index: loncom/interface/loncoursedata.pm
diff -u loncom/interface/loncoursedata.pm:1.104 loncom/interface/loncoursedata.pm:1.105
--- loncom/interface/loncoursedata.pm:1.104	Fri Oct 17 17:36:10 2003
+++ loncom/interface/loncoursedata.pm	Mon Oct 20 16:42:39 2003
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: loncoursedata.pm,v 1.104 2003/10/17 21:36:10 matthew Exp $
+# $Id: loncoursedata.pm,v 1.105 2003/10/20 20:42:39 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -1987,8 +1987,36 @@
     return ();
 }
 
-#sub get_timestamp_data {
-#    my ($students,$symb,
+sub get_student_data {
+    my ($students,$courseid) = @_;
+    $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+    &setup_table_names($courseid);
+    my $dbh = &Apache::lonmysql::get_dbh();
+    return undef if (! defined($dbh));
+    my $request = 'SELECT '.
+        'student_id, student '.
+        'FROM '.$student_table;
+    if (defined($students)) {
+        $request .= ' WHERE ('.
+            join(' OR ', map {'student_id='.
+                                  &get_student_id($_->{'username'},
+                                                  $_->{'domain'})
+                              } @$students
+                 ).')';
+    }
+    $request.= ' ORDER BY student_id';
+    my $sth = $dbh->prepare($request);
+    $sth->execute();
+    if ($dbh->err) {
+        &Apache::lonnet::logthis('error = '.$dbh->errstr());
+        return undef;
+    }
+    my $dataset = $sth->fetchall_arrayref();
+    if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
+        return $dataset;
+    }
+}
+
 sub get_optionresponse_data {
     my ($students,$symb,$response,$courseid) = @_;
     return undef if (! defined($symb) || 
@@ -2002,8 +2030,8 @@
     my $dbh = &Apache::lonmysql::get_dbh();
     return undef if (! defined($dbh));
     my $request = 'SELECT '.
-        'a.awarddetail, a.response_specific_value, a.submission, '.
-        'b.timestamp, c.tries '.
+        'a.student_id, a.awarddetail, a.response_specific_value, '.
+        'a.submission, b.timestamp, c.tries '.
         'FROM '.$fulldump_response_table.' AS a '.
         'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
         'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
@@ -2025,6 +2053,10 @@
 #    &Apache::lonnet::logthis("request =\n".$request);
     my $sth = $dbh->prepare($request);
     $sth->execute();
+    if ($dbh->err) {
+        &Apache::lonnet::logthis('error = '.$dbh->errstr());
+        return undef;
+    }
     my $dataset = $sth->fetchall_arrayref();
     if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
         return $dataset;
Index: loncom/interface/statistics/lonproblemanalysis.pm
diff -u loncom/interface/statistics/lonproblemanalysis.pm:1.42 loncom/interface/statistics/lonproblemanalysis.pm:1.43
--- loncom/interface/statistics/lonproblemanalysis.pm:1.42	Fri Oct 17 18:05:45 2003
+++ loncom/interface/statistics/lonproblemanalysis.pm	Mon Oct 20 16:42:39 2003
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonproblemanalysis.pm,v 1.42 2003/10/17 22:05:45 matthew Exp $
+# $Id: lonproblemanalysis.pm,v 1.43 2003/10/20 20:42:39 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -36,6 +36,7 @@
 use Apache::lonlocal;
 use HTML::Entities();
 use Time::Local();
+use Spreadsheet::WriteExcel();
 
 my $plotcolors = ['#33ff00', 
                   '#0033cc', '#990000', '#aaaa66', '#663399', '#ff9933',
@@ -77,18 +78,32 @@
         $r->print('<input type="submit" name="SelectAnother" value="'.
                   &mt('Choose a different resource').'" />');
         $r->print('&nbsp;'x5);
+        $r->print('<input type="submit" name="ExcelOutput" value="'.
+                  &mt('Produce Excel Data Sheet').'" />');
+        $r->print('&nbsp;'x5);
         #
         $r->print('<hr />');
         #
         my ($symb,$part,$resid) = &get_problem_symb(
                      &Apache::lonnet::unescape($ENV{'form.problemchoice'})
                                            );
+        $r->rflush();
         #
         my $resource = &get_resource_from_symb($symb);
         if (defined($resource)) {
             $r->print('<h3>'.$resource->{'src'}.'</h3>');
             my %Data = &get_problem_data($resource->{'src'});
+            my $PerformanceData = 
+                &Apache::loncoursedata::get_optionresponse_data
+                                           (\@Students,$symb,$resid);
             my $ORdata = $Data{$part.'.'.$resid};
+            if (exists($ENV{'form.ExcelOutput'}) && 
+                defined($PerformanceData)) {
+                my $result = &prepare_excel_sheet($r,$resource,
+                                                  $PerformanceData,$ORdata);
+                $r->print($result);
+                $r->rflush();
+            }
             ##
             ## Render the problem
             my $base;
@@ -102,21 +117,21 @@
                       '<base href="'.$base.'" />'.
                       $rendered_problem.
                       '</td></tr></table>');
+            $r->rflush();
             ##
             ## Analyze the problem
-            my $PerformanceData = 
-                &Apache::loncoursedata::get_optionresponse_data
-                                           (\@Students,$symb,$resid);
             if (defined($PerformanceData) && 
                 ref($PerformanceData) eq 'ARRAY') {
                 if ($ENV{'form.AnalyzeOver'} eq 'Tries') {
-                    my $analysis_html = &tries_analysis($PerformanceData,
+                    my $analysis_html = &tries_analysis($r,$PerformanceData,
                                                          $ORdata);
                     $r->print($analysis_html);
+                    $r->rflush();
                 } elsif ($ENV{'form.AnalyzeOver'} eq 'Time') {
                     my $analysis_html = &time_analysis($PerformanceData,
                                                          $ORdata);
-                $r->print($analysis_html);
+                    $r->print($analysis_html);
+                    $r->rflush();
                 } else {
                     $r->print('<h2>'.
                               &mt('The analysis you have selected is '.
@@ -173,13 +188,13 @@
                        eight => 8,
                        nine  => 9,
                        ten   => 10,);
-        my $a1 = $a; 
-        my $b1 = $b;
-        if (exists($Numbers{lc($a)})) {
-            $a1 = $Numbers{lc($a)};
+        my $a1 = lc($a); 
+        my $b1 = lc($b);
+        if (exists($Numbers{$a})) {
+            $a1 = $Numbers{$a};
         }
-        if (exists($Numbers{lc($b)})) {
-            $b1 = $Numbers{lc($b)};
+        if (exists($Numbers{$b})) {
+            $b1 = $Numbers{$b};
         }
         $a1 cmp $b1;
     };
@@ -287,7 +302,7 @@
 #########################################################
 #########################################################
 sub tries_analysis {
-    my ($PerformanceData,$ORdata) = @_;
+    my ($r,$PerformanceData,$ORdata) = @_;
     my $mintries = 1;
     my $maxtries = $ENV{'form.NumPlots'};
     my ($table,$Foils,$Concepts) = &build_foil_index($ORdata);
@@ -298,7 +313,7 @@
             '</h3>'.$table;
         $ENV{'form.AnalyzeAs'} = 'Foils';
     }
-    my %ResponseData = &analyze_option_data_by_tries($PerformanceData,
+    my %ResponseData = &analyze_option_data_by_tries($r,$PerformanceData,
                                                      $mintries,$maxtries);
     my $analysis = '';
     if ($ENV{'form.AnalyzeAs'} eq 'Foils') {
@@ -460,7 +475,7 @@
 }
 
 sub analyze_option_data_by_tries {
-    my ($PerformanceData,$mintries,$maxtries) = @_;
+    my ($r,$PerformanceData,$mintries,$maxtries) = @_;
     my %Trydata;
     $mintries = 1         if (! defined($mintries) || $mintries < 1);
     $maxtries = $mintries if (! defined($maxtries) || $maxtries < $mintries);
@@ -468,6 +483,7 @@
         next if (! defined($row));
         my $tries = &get_tries_from_row($row);
         my %Row   = &Process_Row($row);
+        next if (! %Row);
         while (my ($foilid,$href) = each(%Row)) {
             if (! ref($href)) { 
                 $Trydata{$foilid}->[$tries] += $href;
@@ -719,13 +735,394 @@
 ##
 #########################################################
 #########################################################
+
 sub prepare_excel_sheet {
-    my ($PerformanceData,$ORdata) = @_;
+    my ($r,$resource,$PerformanceData,$ORdata) = @_;
+    my $response = '<h2>'.&mt('Excel Spreadsheet Creation').'</h2>';
     my (undef,$Foils,$Concepts) = &build_foil_index($ORdata);
+    #
+    # Create excel worksheet
+    my $filename = '/prtspool/'.
+        $ENV{'user.name'}.'_'.$ENV{'user.domain'}.'_'.
+        time.'_'.rand(1000000000).'.xls';
+    my $workbook  = Spreadsheet::WriteExcel->new('/home/httpd'.$filename);
+    if (! defined($workbook)) {
+        $r->log_error("Error creating excel spreadsheet $filename: $!");
+        $r->print('<p>'.&mt("Unable to create new Excel file.  ".
+                            "This error has been logged.  ".
+                            "Please alert your LON-CAPA administrator").
+                  '</p>');
+        return undef;
+    }
+    #
+    $workbook->set_tempdir('/home/httpd/perl/tmp');
+    #
+    # Define some potentially useful formats
+    my $format;
+    $format->{'header'} = $workbook->add_format(bold      => 1, 
+                                                bottom    => 1,
+#                                                text_wrap => 1,
+                                                align     => 'center');
+    $format->{'bold'} = $workbook->add_format(bold=>1);
+    $format->{'h1'}   = $workbook->add_format(bold=>1, size=>18);
+    $format->{'h2'}   = $workbook->add_format(bold=>1, size=>16);
+    $format->{'h3'}   = $workbook->add_format(bold=>1, size=>14);
+    $format->{'date'} = $workbook->add_format(num_format=>
+                                              'mmm d yyyy hh:mm AM/PM');
+    #
+    # Create and populate main worksheets
+    my $problem_data_sheet  = $workbook->addworksheet('Problem Data');
+    my $student_data_sheet  = $workbook->addworksheet('Student Data');
+    my $response_data_sheet = $workbook->addworksheet('Response Data');
+    foreach my $sheet ($problem_data_sheet,$student_data_sheet,
+                       $response_data_sheet) {
+        $sheet->write(0,0,$resource->{'title'},$format->{'h2'});
+        $sheet->write(1,0,$resource->{'src'},$format->{'h3'});
+    }
+    #
+    my $result;
+    $result = &build_problem_data_worksheet($problem_data_sheet,$format,
+                                            $Concepts,$ORdata);
+    if ($result ne 'okay') {
+        # Do something useful
+    }
+    $result = &build_student_data_worksheet($student_data_sheet,$format);
+    if ($result ne 'okay') {
+        # Do something useful
+    }
+    $result = &build_response_data_worksheet($response_data_sheet,$format,
+                                             $PerformanceData,$Foils,
+                                             $ORdata);
+    if ($result ne 'okay') {
+        # Do something useful
+    }
+   $response_data_sheet->activate();
+    #
+    # Close the excel file
+    $workbook->close();
+    #
+    # Write a link to allow them to download it
+    $result .= '<br />'.
+              '<a href="'.$filename.'">Your Excel spreadsheet.</a>'."\n";
+    return $result;
+}
+
+sub build_problem_data_worksheet {
+    my ($worksheet,$format,$Concepts,$ORdata) = @_;
+    my $rows_output = 3;
+    my $cols_output = 0;
+    $worksheet->write($rows_output++,0,'Problem Structure',$format->{'h3'});
+    ##
+    ##
+    my @Headers;
+    if (@$Concepts > 1) {
+        @Headers = ("Concept\nNumber",'Concept',"Foil\nNumber",
+                    'Foil Name','Foil Text','Correct value');
+    } else {
+        @Headers = ('Foil Number','FoilName','Foil Text','Correct value');
+    }
+    $worksheet->write_row($rows_output++,0,\@Headers,$format->{'header'});
+    my %Foildata = %{$ORdata->{'Foils'}};
+    my $conceptindex = 1;
+    my $foilindex = 1;
+    foreach my $concept (@$Concepts) {
+        my @FoilsInConcept = @{$concept->{'foils'}};
+        my $firstfoil = shift(@FoilsInConcept);
+        if (@$Concepts > 1) {
+            $worksheet->write_row($rows_output++,0,
+                                  [$conceptindex,
+                                   $concept->{'name'},
+                                   $foilindex++,
+                                   $Foildata{$firstfoil}->{'name'},
+                                   $Foildata{$firstfoil}->{'text'},
+                                   $Foildata{$firstfoil}->{'value'},]);
+        } else {
+            $worksheet->write_row($rows_output++,0,
+                                  [ $foilindex++,
+                                    $Foildata{$firstfoil}->{'name'},
+                                    $Foildata{$firstfoil}->{'text'},
+                                    $Foildata{$firstfoil}->{'value'},]);
+        }
+        foreach my $foilid (@FoilsInConcept) {
+            if (@$Concepts > 1) {
+                $worksheet->write_row($rows_output++,0,
+                                      ['',
+                                       '',
+                                       $foilindex,
+                                       $Foildata{$foilid}->{'name'},
+                                       $Foildata{$foilid}->{'text'},
+                                       $Foildata{$foilid}->{'value'},]);
+            } else {
+                $worksheet->write_row($rows_output++,0,                
+                                      [$foilindex,
+                                       $Foildata{$foilid}->{'name'},
+                                       $Foildata{$foilid}->{'text'},
+                                       $Foildata{$foilid}->{'value'},]);
+            }                
+        } continue {
+            $foilindex++;
+        }
+    } continue {
+        $conceptindex++;
+    }
+    $rows_output++;
+    $rows_output++;
+    ##
+    ## Option data output
+    $worksheet->write($rows_output++,0,'Options',$format->{'header'});
+    foreach my $string (@{$ORdata->{'Options'}}) {
+        $worksheet->write($rows_output++,0,$string);
+    }
+    return 'okay';
+}
+
+sub build_student_data_worksheet {
+    my ($worksheet,$format) = @_;
+    my $rows_output = 3;
+    my $cols_output = 0;
+    $worksheet->write($rows_output++,0,'Student Data',$format->{'h3'});
+    my @Headers = ('full name','username','domain','section',
+                   "student\nnumber",'identifier');
+    $worksheet->write_row($rows_output++,0,\@Headers,$format->{'header'});
+    my @Students = @Apache::lonstatistics::Students;
+    my $studentrows = &Apache::loncoursedata::get_student_data(\@Students);
+    my %ids;
+    foreach my $row (@$studentrows) {
+        my ($mysqlid,$student) = @$row;
+        $ids{$student}=$mysqlid;
+    }
+    foreach my $student (@Students) {
+        my $name_domain = $student->{'username'}.':'.$student->{'domain'};
+        $worksheet->write_row($rows_output++,0,
+                          [$student->{'fullname'},
+                           $student->{'username'},$student->{'domain'},
+                           $student->{'section'},$student->{'id'},
+                           $ids{$name_domain}]);
+    }
+    return;
+}
+
+sub build_response_data_worksheet {
+    my ($worksheet,$format,$PerformanceData,$Foils,$ORdata)=@_;
+    my $rows_output = 3;
+    my $cols_output = 0;
+    $worksheet->write($rows_output++,0,'Response Data',$format->{'h3'});
+    $worksheet->set_column(1,1,20);
+    $worksheet->set_column(2,2,13);
+    my @Headers = ('identifier','time','award detail','attempt');
+    foreach my $foil (@$Foils) {
+        push (@Headers,$foil.' submission');
+        push (@Headers,$foil.' grading');
+    }
+    $worksheet->write_row($rows_output++,0,\@Headers,$format->{'header'});
+    #
+    foreach my $row (@$PerformanceData) {
+        next if (! defined($row));
+        my ($student,$award,$grading,$submission,$time,$tries) = @$row;
+        my @Foilgrades = split('&',$grading);
+        my @Foilsubs   = split('&',$submission);
+        my %ResponseData;
+        for (my $j=0;$j<=$#Foilgrades;$j++) {
+            my ($foilid,$correct)  = split('=',$Foilgrades[$j]);
+            my (undef,$submission) = split('=',$Foilsubs[$j]);
+            $submission = &Apache::lonnet::unescape($submission);
+            $ResponseData{$foilid.' submission'}=$submission;
+            $ResponseData{$foilid.' award'}=$correct;
+        }
+        $worksheet->write($rows_output,$cols_output++,$student);
+        $worksheet->write($rows_output,$cols_output++,
+                          &calc_serial($time),$format->{'date'});
+        $worksheet->write($rows_output,$cols_output++,$award);
+        $worksheet->write($rows_output,$cols_output++,$tries);
+        foreach my $foilid (@$Foils) {
+            $worksheet->write($rows_output,$cols_output++,
+                              $ResponseData{$foilid.' submission'});
+            $worksheet->write($rows_output,$cols_output++,
+                              $ResponseData{$foilid.' award'});
+        }
+        $rows_output++;
+        $cols_output = 0;
+    }
+    return;
 }
 
-sub process_data_for_excel_sheet {
-    my ($PerformanceData,$Foils,$Concepts,$ORdata)=@_;
+
+##
+## The following is copied from datecalc1.pl, part of the 
+## Spreadsheet::WriteExcel CPAN module.
+##
+##
+######################################################################
+#
+# Demonstration of writing date/time cells to Excel spreadsheets,
+# using UNIX/Perl time as source of date/time.
+#
+# Copyright 2000, Andrew Benham, adsb@bigfoot.com
+#
+######################################################################
+#
+# UNIX/Perl time is the time since the Epoch (00:00:00 GMT, 1 Jan 1970)
+# measured in seconds.
+#
+# An Excel file can use exactly one of two different date/time systems.
+# In these systems, a floating point number represents the number of days
+# (and fractional parts of the day) since a start point. The floating point
+# number is referred to as a 'serial'.
+# The two systems ('1900' and '1904') use different starting points:
+#  '1900'; '1.00' is 1 Jan 1900 BUT 1900 is erroneously regarded as
+#          a leap year - see:
+#            http://support.microsoft.com/support/kb/articles/Q181/3/70.asp
+#          for the excuse^H^H^H^H^H^Hreason.
+#  '1904'; '1.00' is 2 Jan 1904.
+#
+# The '1904' system is the default for Apple Macs. Windows versions of
+# Excel have the option to use the '1904' system.
+#
+# Note that Visual Basic's "DateSerial" function does NOT erroneously
+# regard 1900 as a leap year, and thus its serials do not agree with
+# the 1900 serials of Excel for dates before 1 Mar 1900.
+#
+# Note that StarOffice (at least at version 5.2) does NOT erroneously
+# regard 1900 as a leap year, and thus its serials do not agree with
+# the 1900 serials of Excel for dates before 1 Mar 1900.
+#
+######################################################################
+#
+# Calculation description
+# =======================
+#
+# 1900 system
+# -----------
+# Unix time is '0' at 00:00:00 GMT 1 Jan 1970, i.e. 70 years after 1 Jan 1900.
+# Of those 70 years, 17 (1904,08,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68)
+# were leap years with an extra day.
+# Thus there were 17 + 70*365 days = 25567 days between 1 Jan 1900 and
+# 1 Jan 1970.
+# In the 1900 system, '1' is 1 Jan 1900, but as 1900 was not a leap year
+# 1 Jan 1900 should really be '2', so 1 Jan 1970 is '25569'.
+#
+# 1904 system
+# -----------
+# Unix time is '0' at 00:00:00 GMT 1 Jan 1970, i.e. 66 years after 1 Jan 1904.
+# Of those 66 years, 17 (1904,08,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68)
+# were leap years with an extra day.
+# Thus there were 17 + 66*365 days = 24107 days between 1 Jan 1904 and
+# 1 Jan 1970.
+# In the 1904 system, 2 Jan 1904 being '1', 1 Jan 1970 is '24107'.
+#
+######################################################################
+#
+# Copyright (c) 2000, Andrew Benham.
+# This program is free software. It may be used, redistributed and/or
+# modified under the same terms as Perl itself.
+#
+# Andrew Benham, adsb@bigfoot.com
+# London, United Kingdom
+# 11 Nov 2000
+#
+######################################################################
+
+# Use 1900 date system on all platforms other than Apple Mac (for which
+# use 1904 date system).
+my $DATE_SYSTEM = ($^O eq 'MacOS') ? 1 : 0;
+
+#-----------------------------------------------------------
+# calc_serial()
+#
+# Called with (up to) 2 parameters.
+#   1.  Unix timestamp.  If omitted, uses current time.
+#   2.  GMT flag. Set to '1' to return serial in GMT.
+#       If omitted, returns serial in appropriate timezone.
+#
+# Returns date/time serial according to $DATE_SYSTEM selected
+#-----------------------------------------------------------
+sub calc_serial {
+        my $time = (defined $_[0]) ? $_[0] : time();
+        my $gmtflag = (defined $_[1]) ? $_[1] : 0;
+
+        # Divide timestamp by number of seconds in a day.
+        # This gives a date serial with '0' on 1 Jan 1970.
+        my $serial = $time / 86400;
+
+        # Adjust the date serial by the offset appropriate to the
+        # currently selected system (1900/1904).
+        if ($DATE_SYSTEM == 0) {        # use 1900 system
+                $serial += 25569;
+        } else {                        # use 1904 system
+                $serial += 24107;
+        }
+
+        unless ($gmtflag) {
+                # Now have a 'raw' serial with the right offset. But this
+                # gives a serial in GMT, which is false unless the timezone
+                # is GMT. We need to adjust the serial by the appropriate
+                # timezone offset.
+                # Calculate the appropriate timezone offset by seeing what
+                # the differences between localtime and gmtime for the given
+                # time are.
+
+                my @gmtime = gmtime($time);
+                my @ltime  = localtime($time);
+
+                # For the first 7 elements of the two arrays, adjust the
+                # date serial where the elements differ.
+                for (0 .. 6) {
+                        my $diff = $ltime[$_] - $gmtime[$_];
+                        if ($diff) {
+                                $serial += _adjustment($diff,$_);
+                        }
+                }
+        }
+
+        # Perpetuate the error that 1900 was a leap year by decrementing
+        # the serial if we're using the 1900 system and the date is prior to
+        # 1 Mar 1900. This has the effect of making serial value '60'
+        # 29 Feb 1900.
+
+        # This fix only has any effect if UNIX/Perl time on the platform
+        # can represent 1900. Many can't.
+
+        unless ($DATE_SYSTEM) {
+                $serial-- if ($serial < 61);    # '61' is 1 Mar 1900
+        }
+        return $serial;
+}
+
+sub _adjustment {
+        # Based on the difference in the localtime/gmtime array elements
+        # number, return the adjustment required to the serial.
+
+        # We only look at some elements of the localtime/gmtime arrays:
+        #    seconds    unlikely to be different as all known timezones
+        #               have an offset of integral multiples of 15 minutes,
+        #               but it's easy to do.
+        #    minutes    will be different for timezone offsets which are
+        #               not an exact number of hours.
+        #    hours      very likely to be different.
+        #    weekday    will differ when localtime/gmtime difference
+        #               straddles midnight.
+        #
+        # Assume that difference between localtime and gmtime is less than
+        # 5 days, then don't have to do maths for day of month, month number,
+        # year number, etc...
+
+        my ($delta,$element) = @_;
+        my $adjust = 0;
+
+        if ($element == 0) {            # Seconds
+                $adjust = $delta/86400;         # 60 * 60 * 24
+        } elsif ($element == 1) {       # Minutes
+                $adjust = $delta/1440;          # 60 * 24
+        } elsif ($element == 2) {       # Hours
+                $adjust = $delta/24;            # 24
+        } elsif ($element == 6) {       # Day of week number
+                # Catch difference straddling Sat/Sun in either direction
+                $delta += 7 if ($delta < -4);
+                $delta -= 7 if ($delta > 4);
+
+                $adjust = $delta;
+        }
+        return $adjust;
 }
 
 #########################################################
@@ -910,7 +1307,7 @@
 sub get_tries_from_row {
     my ($row) = @_;
     if (ref($row)) {
-        return $row->[4];
+        return $row->[5];
     }
     return undef;
 }
@@ -918,8 +1315,8 @@
 sub Process_Row {
     my ($row) = @_;
     my %RowData;
-    my ($award,$grading,$submission,$time,$tries) = @$row;
-    next if ($award eq 'MISSING_ANSWER');
+    my ($student_id,$award,$grading,$submission,$time,$tries) = @$row;
+    return undef if ($award eq 'MISSING_ANSWER');
     if ($award =~ /(APPROX_ANS|EXACT_ANS)/) {
         $RowData{'_correct'} = 1;
     }
@@ -933,7 +1330,6 @@
             $RowData{$foilid}->{'_correct'}++;
         } else {
             $submission = &Apache::lonnet::unescape($submission);
-            $submission =~ s/\%20/ /g;
             $RowData{$foilid}->{$submission}++;
         }
         $RowData{$foilid}->{'_total'}++;
@@ -973,6 +1369,7 @@
                 if ($key=~ /^foil\.text\.(.*)$/) {
                     my $foil = $1;
                     $Partdata{$part}->{'Foils'}->{$foil}->{'name'}=$foil;
+                    $value =~ s/(\s*$|^\s*)//g;
                     $Partdata{$part}->{'Foils'}->{$foil}->{'text'}=$value;
                 } elsif ($key =~ /^foil\.value\.(.*)$/) {
                     my $foil = $1;

--matthew1066682559--