[LON-CAPA-cvs] cvs: loncom(version_1_3_X_memcached) /lonnet/perl lonnet.pm
albertel
lon-capa-cvs@mail.lon-capa.org
Thu, 10 Feb 2005 08:16:32 -0000
This is a MIME encoded message
--albertel1108023392
Content-Type: text/plain
albertel Thu Feb 10 03:16:32 2005 EDT
Modified files: (Branch: version_1_3_X_memcached)
/loncom/lonnet/perl lonnet.pm
Log:
- forward porting the memcached changes
--albertel1108023392
Content-Type: text/plain
Content-Disposition: attachment; filename="albertel-20050210031632.txt"
Index: loncom/lonnet/perl/lonnet.pm
diff -u loncom/lonnet/perl/lonnet.pm:1.587.2.3 loncom/lonnet/perl/lonnet.pm:1.587.2.3.2.1
--- loncom/lonnet/perl/lonnet.pm:1.587.2.3 Fri Jan 28 04:26:28 2005
+++ loncom/lonnet/perl/lonnet.pm Thu Feb 10 03:16:31 2005
@@ -1,7 +1,7 @@
# The LearningOnline Network
# TCP networking package
#
-# $Id: lonnet.pm,v 1.587.2.3 2005/01/28 09:26:28 albertel Exp $
+# $Id: lonnet.pm,v 1.587.2.3.2.1 2005/02/10 08:16:31 albertel Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -36,7 +36,7 @@
# use Date::Parse;
use vars
qw(%perlvar %hostname %homecache %badServerCache %hostip %iphost %spareid %hostdom
- %libserv %pr %prp %metacache %packagetab %titlecache %courseresversioncache %resversioncache
+ %libserv %pr %prp $metacache %packagetab %titlecache %courseresversioncache %resversioncache
%courselogs %accesshash %userrolehash $processmarker $dumpcount
%coursedombuf %coursenumbuf %coursehombuf %coursedescrbuf %courseinstcodebuf %courseownerbuf %courseresdatacache
%userresdatacache %getsectioncache %domaindescription %domain_auth_def %domain_auth_arg_def
@@ -50,6 +50,7 @@
use Apache::lonlocal;
use Storable qw(lock_store lock_nstore lock_retrieve freeze thaw nfreeze);
use Time::HiRes qw( gettimeofday tv_interval );
+use Cache::Memcached;
my $readit;
my $max_connection_retries = 10; # Or some such value.
@@ -1020,6 +1021,73 @@
# &logthis("load_cache_item $name took ".(&Time::HiRes::time()-$starttime));
}
+sub devalidate_cache_new {
+ my ($cache,$name,$id) = @_;
+ if (1) { &Apache::lonnet::logthis("deleting $name:$id"); }
+ $cache->delete(&escape($name.':'.$id));
+}
+
+my $to_remember=10;
+my %remembered;
+my %accessed;
+sub is_cached_new {
+ my ($cache,$name,$id,$debug) = @_;
+ $debug=0;
+ $id=&escape($name.':'.$id);
+ if (exists($remembered{$id})) {
+ if ($debug) { &Apache::lonnet::logthis("Earyl return $id of $remembered{$id} "); }
+ $accessed{$id}=[&gettimeofday()];
+ return ($remembered{$id},1);
+ }
+ my $value = $cache->get($id);
+ if (!(defined($value))) {
+ if ($debug) { &Apache::lonnet::logthis("getting $id is not defined"); }
+ return (undef,undef);
+ }
+ &make_room($id,$value);
+ if ($value eq '__undef__') {
+ if ($debug) { &Apache::lonnet::logthis("getting $id is __undef__"); }
+ return (undef,1);
+ }
+ if ($debug) { &Apache::lonnet::logthis("getting $id is $value"); }
+ return ($value,1);
+}
+
+sub do_cache_new {
+ my ($cache,$name,$id,$value,$time,$debug) = @_;
+ $debug=0;
+ $id=&escape($name.':'.$id);
+ my $setvalue=$value;
+ if (!defined($setvalue)) {
+ $setvalue='__undef__';
+ }
+ if ($debug) { &Apache::lonnet::logthis("Setting $id to $value"); }
+ $cache->set($id,$setvalue,300);
+ return $value;
+}
+
+my $kicks=0;
+sub make_room {
+ my ($id,$value)=@_;
+ my $debug=0;
+ $remembered{$id}=$value;
+ $accessed{$id}=[&gettimeofday()];
+ if (scalar(keys(%remembered)) <= $to_remember) { return; }
+ my $to_kick;
+ my $max_time=0;
+ foreach my $other (keys(%accessed)) {
+ if (&tv_interval($accessed{$other}) > $max_time) {
+ $to_kick=$other;
+ $max_time=&tv_interval($accessed{$other});
+ }
+ }
+ delete($remembered{$to_kick});
+ delete($accessed{$to_kick});
+ $kicks++;
+ if ($debug) { &logthis("kicking $max_time $kicks\n"); }
+ return;
+}
+
# ------------------------------------- Read an entry from a user's environment
sub userenvironment {
@@ -4464,6 +4532,7 @@
# ---------------------------------------------------------------- Get metadata
+my %metaentry;
sub metadata {
my ($uri,$what,$liburi,$prefix,$depthcount)=@_;
$uri=&declutter($uri);
@@ -4483,28 +4552,29 @@
# Everything is cached by the main uri, libraries are never directly cached
#
if (!defined($liburi)) {
- my ($result,$cached)=&is_cached(\%metacache,$uri,'meta');
+ my ($result,$cached)=&is_cached_new($metacache,'meta',$uri);
if (defined($cached)) { return $result->{':'.$what}; }
}
{
#
# Is this a recursive call for a library?
#
- if (! exists($metacache{$uri})) {
- $metacache{$uri}={};
- }
+# if (! exists($metacache{$uri})) {
+# $metacache{$uri}={};
+# }
if ($liburi) {
$liburi=&declutter($liburi);
$filename=$liburi;
} else {
- &devalidate_cache(\%metacache,$uri,'meta');
+ &devalidate_cache_new($metacache,'meta',$uri);
+ undef(%metaentry);
}
my %metathesekeys=();
unless ($filename=~/\.meta$/) { $filename.='.meta'; }
my $metastring;
if ($uri !~ m|^uploaded/|) {
my $file=&filelocation('',&clutter($filename));
- push(@{$metacache{$uri.'.file'}},$file);
+ #push(@{$metaentry{$uri.'.file'}},$file);
$metastring=&getfile($file);
}
my $parser=HTML::LCParser->new(\$metastring);
@@ -4521,10 +4591,10 @@
if (defined($token->[2]->{'id'})) {
$keyroot.='_'.$token->[2]->{'id'};
}
- if ($metacache{$uri}->{':packages'}) {
- $metacache{$uri}->{':packages'}.=','.$package.$keyroot;
+ if ($metaentry{':packages'}) {
+ $metaentry{':packages'}.=','.$package.$keyroot;
} else {
- $metacache{$uri}->{':packages'}=$package.$keyroot;
+ $metaentry{':packages'}=$package.$keyroot;
}
foreach (keys %packagetab) {
my $part=$keyroot;
@@ -4546,14 +4616,14 @@
if ($subp eq 'display') {
$value.=' [Part: '.$part.']';
}
- $metacache{$uri}->{':'.$unikey.'.part'}=$part;
+ $metaentry{':'.$unikey.'.part'}=$part;
$metathesekeys{$unikey}=1;
- unless (defined($metacache{$uri}->{':'.$unikey.'.'.$subp})) {
- $metacache{$uri}->{':'.$unikey.'.'.$subp}=$value;
+ unless (defined($metaentry{':'.$unikey.'.'.$subp})) {
+ $metaentry{':'.$unikey.'.'.$subp}=$value;
}
- if (defined($metacache{$uri}->{':'.$unikey.'.default'})) {
- $metacache{$uri}->{':'.$unikey}=
- $metacache{$uri}->{':'.$unikey.'.default'};
+ if (defined($metaentry{':'.$unikey.'.default'})) {
+ $metaentry{':'.$unikey}=
+ $metaentry{':'.$unikey.'.default'};
}
}
}
@@ -4586,7 +4656,7 @@
foreach (sort(split(/\,/,&metadata($uri,'keys',
$location,$unikey,
$depthcount+1)))) {
- $metacache{$uri}->{':'.$_}=$metacache{$uri}->{':'.$_};
+ $metaentry{':'.$_}=$metaentry{':'.$_};
$metathesekeys{$_}=1;
}
}
@@ -4597,18 +4667,18 @@
}
$metathesekeys{$unikey}=1;
foreach (@{$token->[3]}) {
- $metacache{$uri}->{':'.$unikey.'.'.$_}=$token->[2]->{$_};
+ $metaentry{':'.$unikey.'.'.$_}=$token->[2]->{$_};
}
my $internaltext=&HTML::Entities::decode($parser->get_text('/'.$entry));
- my $default=$metacache{$uri}->{':'.$unikey.'.default'};
+ my $default=$metaentry{':'.$unikey.'.default'};
if ( $internaltext =~ /^\s*$/ && $default !~ /^\s*$/) {
# only ws inside the tag, and not in default, so use default
# as value
- $metacache{$uri}->{':'.$unikey}=$default;
+ $metaentry{':'.$unikey}=$default;
} else {
# either something interesting inside the tag or default
# uninteresting
- $metacache{$uri}->{':'.$unikey}=$internaltext;
+ $metaentry{':'.$unikey}=$internaltext;
}
# end of not-a-package not-a-library import
}
@@ -4625,7 +4695,7 @@
&metadata_create_package_def($uri,$key,'extension_'.$extension,
\%metathesekeys);
}
- if (!exists($metacache{$uri}->{':packages'})) {
+ if (!exists($metaentry{':packages'})) {
foreach my $key (sort(keys(%packagetab))) {
#no specific packages well let's get default then
if ($key!~/^default&/) { next; }
@@ -4634,31 +4704,31 @@
}
}
# are there custom rights to evaluate
- if ($metacache{$uri}->{':copyright'} eq 'custom') {
+ if ($metaentry{':copyright'} eq 'custom') {
#
# Importing a rights file here
#
unless ($depthcount) {
- my $location=$metacache{$uri}->{':customdistributionfile'};
+ my $location=$metaentry{':customdistributionfile'};
my $dir=$filename;
$dir=~s|[^/]*$||;
$location=&filelocation($dir,$location);
foreach (sort(split(/\,/,&metadata($uri,'keys',
$location,'_rights',
$depthcount+1)))) {
- $metacache{$uri}->{':'.$_}=$metacache{$uri}->{':'.$_};
+ #$metaentry{':'.$_}=$metacache{$uri}->{':'.$_};
$metathesekeys{$_}=1;
}
}
}
- $metacache{$uri}->{':keys'}=join(',',keys %metathesekeys);
- &metadata_generate_part0(\%metathesekeys,$metacache{$uri},$uri);
- $metacache{$uri}->{':allpossiblekeys'}=join(',',keys %metathesekeys);
- &do_cache(\%metacache,$uri,$metacache{$uri},'meta');
+ $metaentry{':keys'}=join(',',keys %metathesekeys);
+ &metadata_generate_part0(\%metathesekeys,\%metaentry,$uri);
+ $metaentry{':allpossiblekeys'}=join(',',keys %metathesekeys);
+ &do_cache_new($metacache,'meta',$uri,\%metaentry);
# this is the end of "was not already recently cached
}
- return $metacache{$uri}->{':'.$what};
+ return $metaentry{':'.$what};
}
sub metadata_create_package_def {
@@ -4666,22 +4736,22 @@
my ($pack,$name,$subp)=split(/\&/,$key);
if ($subp eq 'default') { next; }
- if (defined($metacache{$uri}->{':packages'})) {
- $metacache{$uri}->{':packages'}.=','.$package;
+ if (defined($metaentry{':packages'})) {
+ $metaentry{':packages'}.=','.$package;
} else {
- $metacache{$uri}->{':packages'}=$package;
+ $metaentry{':packages'}=$package;
}
my $value=$packagetab{$key};
my $unikey;
$unikey='parameter_0_'.$name;
- $metacache{$uri}->{':'.$unikey.'.part'}=0;
+ $metaentry{':'.$unikey.'.part'}=0;
$$metathesekeys{$unikey}=1;
- unless (defined($metacache{$uri}->{':'.$unikey.'.'.$subp})) {
- $metacache{$uri}->{':'.$unikey.'.'.$subp}=$value;
+ unless (defined($metaentry{':'.$unikey.'.'.$subp})) {
+ $metaentry{':'.$unikey.'.'.$subp}=$value;
}
- if (defined($metacache{$uri}->{':'.$unikey.'.default'})) {
- $metacache{$uri}->{':'.$unikey}=
- $metacache{$uri}->{':'.$unikey.'.default'};
+ if (defined($metaentry{':'.$unikey.'.default'})) {
+ $metaentry{':'.$unikey}=
+ $metaentry{':'.$unikey.'.default'};
}
}
@@ -5573,17 +5643,19 @@
sub goodbye {
&logthis("Starting Shut down");
#not converted to using infrastruture and probably shouldn't be
- &logthis(sprintf("%-20s is %s",'%badServerCache',scalar(%badServerCache)));
+ &logthis(sprintf("%-20s is %s",'%badServerCache',length(&freeze(\%badServerCache))));
#converted
- &logthis(sprintf("%-20s is %s",'%metacache',scalar(%metacache)));
- &logthis(sprintf("%-20s is %s",'%homecache',scalar(%homecache)));
- &logthis(sprintf("%-20s is %s",'%titlecache',scalar(%titlecache)));
- &logthis(sprintf("%-20s is %s",'%courseresdatacache',scalar(%courseresdatacache)));
+# &logthis(sprintf("%-20s is %s",'%metacache',scalar(%metacache)));
+ &logthis(sprintf("%-20s is %s",'%homecache',length(&freeze(\%homecache))));
+ &logthis(sprintf("%-20s is %s",'%titlecache',length(&freeze(\%titlecache))));
+ &logthis(sprintf("%-20s is %s",'%courseresdatacache',length(&freeze(\%courseresdatacache))));
#1.1 only
- &logthis(sprintf("%-20s is %s",'%userresdatacache',scalar(%userresdatacache)));
- &logthis(sprintf("%-20s is %s",'%getsectioncache',scalar(%getsectioncache)));
- &logthis(sprintf("%-20s is %s",'%courseresversioncache',scalar(%courseresversioncache)));
- &logthis(sprintf("%-20s is %s",'%resversioncache',scalar(%resversioncache)));
+ &logthis(sprintf("%-20s is %s",'%userresdatacache',length(&freeze(\%userresdatacache))));
+ &logthis(sprintf("%-20s is %s",'%getsectioncache',length(&freeze(\%getsectioncache))));
+ &logthis(sprintf("%-20s is %s",'%courseresversioncache',length(&freeze(\%courseresversioncache))));
+ &logthis(sprintf("%-20s is %s",'%resversioncache',length(&freeze(\%resversioncache))));
+ &logthis(sprintf("%-20s is %s",'%remembered',length(&freeze(\%remembered))));
+ &logthis(sprintf("%-20s is %s",'kicks',$kicks));
&flushcourselogs();
&logthis("Shutting down");
return DONE;
@@ -5729,7 +5801,7 @@
}
-%metacache=();
+$metacache=new Cache::Memcached({'servers'=>['127.0.0.1:11211']});
$processmarker='_'.time.'_'.$perlvar{'lonHostID'};
$dumpcount=0;
--albertel1108023392--