[LON-CAPA-cvs] cvs: loncom /metadata_database parse_activity_log.pl
matthew
lon-capa-cvs@mail.lon-capa.org
Mon, 11 Jul 2005 05:16:23 -0000
This is a MIME encoded message
--matthew1121058983
Content-Type: text/plain
matthew Mon Jul 11 01:16:23 2005 EDT
Modified files:
/loncom/metadata_database parse_activity_log.pl
Log:
Read xml from a file using a regexp match intead of parsing the XML.
Made xml file an input parameter.
--matthew1121058983
Content-Type: text/plain
Content-Disposition: attachment; filename="matthew-20050711011623.txt"
Index: loncom/metadata_database/parse_activity_log.pl
diff -u loncom/metadata_database/parse_activity_log.pl:1.14 loncom/metadata_database/parse_activity_log.pl:1.15
--- loncom/metadata_database/parse_activity_log.pl:1.14 Thu Mar 31 17:09:40 2005
+++ loncom/metadata_database/parse_activity_log.pl Mon Jul 11 01:16:19 2005
@@ -2,7 +2,7 @@
#
# The LearningOnline Network
#
-# $Id: parse_activity_log.pl,v 1.14 2005/03/31 22:09:40 matthew Exp $
+# $Id: parse_activity_log.pl,v 1.15 2005/07/11 05:16:19 matthew Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -65,13 +65,14 @@
#
# Determine parameters
-my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup);
+my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup,$xmlfile);
&Getopt::Long::GetOptions( "course=s" => \$course,
"domain=s" => \$domain,
"backup" => \$backup,
"help" => \$help,
"logfile=s" => \$logfile,
"srcfile=s" => \$srcfile,
+ "justloadxml=s" => \$xmlfile,
"timerun" => \$time_run,
"nocleanup" => \$nocleanup,
"dropwhendone" => \$drop_when_done,
@@ -153,8 +154,19 @@
my $gz_sql_filename = $sql_filename.'.gz';
#
my $xml_filename = $sourcefilename;
-$xml_filename =~ s|[^/]*$|activity.log.xml|;
my $gz_xml_filename = $xml_filename.'.gz';
+if (defined($xmlfile)) {
+ $xml_filename = $xmlfile;
+ if ($xml_filename =~ /\.gz$/) {
+ $gz_xml_filename = $xml_filename;
+ } else {
+ $gz_xml_filename = $xml_filename.'.gz';
+ }
+} else {
+ my $xml_filename = $sourcefilename;
+ $xml_filename =~ s|[^/]*$|activity.log.xml|;
+ $gz_xml_filename = $xml_filename.'.gz';
+}
#
$error_filename = $sourcefilename;
$error_filename =~ s|[^/]*$|activity.log.errors|;
@@ -163,6 +175,7 @@
#
# Wait for a lock on the lockfile to avoid collisions
my $lockfilename = $sourcefilename.'.lock';
+if (! defined($xmlfile)) {
open(LOCKFILE,'>'.$lockfilename);
if (!flock(LOCKFILE,LOCK_EX)) {
warn("Unable to lock $lockfilename. Aborting".$/);
@@ -192,7 +205,7 @@
}
close(LOCKFILE);
-
+}
##
## Table definitions
##
@@ -345,6 +358,10 @@
}
}
+if (defined($xmlfile)) {
+ exit(0);
+}
+
##
## Ensure the tables we need exist
# create_tables does not complain if the tables already exist
@@ -376,7 +393,8 @@
exit 5;
} elsif ($result > 0) {
$time_this->();
- $logthis->('process_courselog returned '.$result.' backing up tables');
+ $logthis->('process_courselog returned '.$result.'.'.$/.
+ 'Backing up tables');
&backup_tables_as_xml($gz_xml_filename,\%tables);
$time_this->('write backup tables');
}
@@ -405,7 +423,7 @@
}
foreach my $file ($lockfilename, $error_filename,$logfile) {
- if (-z $file) {
+ if (defined($file) && -z $file) {
unlink($file);
}
}
@@ -788,6 +806,10 @@
##
###############################################################
###############################################################
+{
+ my @fields = ('resource','time',
+ 'student','action','idx','machine','action_values');
+ my %ids = ();
sub load_backup_xml_tables {
my ($filename,$tables) = @_;
my $xmlfh;
@@ -795,44 +817,101 @@
if (! defined($xmlfh)) {
return ('error:unable to read '.$filename);
}
- my $dbh = &Apache::lonmysql::get_dbh();
- my $parser = HTML::TokeParser->new($xmlfh);
- $parser->xml_mode('1');
- &store_entry();
+ #
+ %ids = (resource=> {"\0count"=>1},
+ student=> {"\0count"=>1},
+ machine=> {"\0count"=>1});
+ #
my %data;
- while (my $token = $parser->get_token()) {
- if ($token->[0] eq 'S' && $token->[1] eq 'row') {
- undef(%data);
- }
- foreach my $tag ('resource','time','idx',
- 'student','action','machine','action_values') {
- if ($token->[0] eq 'S' && $token->[1] eq $tag) {
- my $text = $parser->get_text("/$tag");
- $data{$tag} = $text;
- }
- }
- if ($token->[0] eq 'E' && $token->[1] eq 'row') {
- $data{'action_values'} =qq{'$data{'action_values'}'};
- my $error = &store_entry($dbh,$tables,\%data);
+ while (my $inputline = <$xmlfh>) {
+ my ($resource,$time,undef,$student,$action,$machine,$action_values) =
+ ($inputline =~ m{<row>
+ <resource>(.*)</resource>
+ <time>(.*)</time>
+ <idx>(.*)</idx>
+ <student>(.*)</student>
+ <action>(.*)</action>
+ <machine>(.*)</machine>
+ <action_values>(.*)</action_values>
+ </row>$
+ }x
+ );
+ my $resource_id = &xml_get_id('resource',$resource);
+ my $student_id = &xml_get_id('student',$student);
+ my $machine_id = &xml_get_id('machine',$machine);
+ &xml_store_activity_row(map { defined($_)?qq{'$_'}:''
+ } ($resource_id,
+ $time,
+ $student_id,
+ $action,
+ 'NULL',
+ $machine_id,
+ $action_values));
+ }
+ &xml_store_activity_row();
+ close($xmlfh);
+ # Store id tables
+ while (my ($id_name,$id_data) = each(%ids)) {
+ if ($id_name eq 'resource') { $id_name = 'res'; }
+ delete($id_data->{"\0count"});
+ &xml_store_id_table($id_name,$id_data);
+ }
+ return;
+}
+
+sub xml_get_id {
+ my ($table,$element) = @_;
+ if (! exists($ids{$table}->{$element})) {
+ $ids{$table}->{$element} = $ids{$table}->{"\0count"}++;
+ }
+ return $ids{$table}->{$element};
+}
+
+{
+ my @data_rows;
+sub xml_store_activity_row {
+ my @data = @_;
+ if (scalar(@data)) {
+ push(@data_rows,[@data]);
+ }
+ if (! scalar(@data) || scalar(@data_rows) > 500) {
+ if (! &Apache::lonmysql::bulk_store_rows($tables{'activity'},
+ scalar(@{$data_rows[0]}),
+ \@data_rows)) {
+ $logthis->("Error:".&Apache::lonmysql::get_error());
+ warn("Error:".&Apache::lonmysql::get_error());
+ } else {
+ undef(@data_rows);
}
}
- &store_entry($dbh,$tables);
return;
}
+}
+
+sub xml_store_id_table {
+ my ($table,$tabledata) =@_;
+ if (! &Apache::lonmysql::bulk_store_rows
+ ($tables{$table},2,
+ [map{[$tabledata->{$_},qq{"$_"}]} keys(%$tabledata)])) {
+ $logthis->("Error:".&Apache::lonmysql::get_error());
+ warn "Error:".&Apache::lonmysql::get_error().$/;
+ }
+}
+
+} # End of load xml scoping
#######################################################################
#######################################################################
##
## store_entry - accumulate data to be inserted into the database
-##
+##
## Pass no values in to clear accumulator
## Pass ($dbh,\%tables) to initiate storage of values
## Pass ($dbh,\%tables,\%data) to use normally
##
#######################################################################
#######################################################################
-
{
my @rows;
my $max_row_count = 100;
@@ -870,11 +949,11 @@
$machine_id,
$data->{'action_values'}]);
}
- if (defined($tables) &&
+ if (defined($tables) &&
( (! defined($data) && scalar(@rows)) || scalar(@rows)>$max_row_count)
){
# Store the rows
- my $result =
+ my $result =
&Apache::lonmysql::bulk_store_rows($tables->{'activity'},
undef,
\@rows);
@@ -886,7 +965,6 @@
undef(@rows);
return $result if (! defined($data));
}
-
return '';
}
--matthew1121058983--