[LON-CAPA-cvs] cvs: loncom /interface lonsearchcat.pm

matthew lon-capa-cvs@mail.lon-capa.org
Fri, 26 Jul 2002 16:37:58 -0000


This is a MIME encoded message

--matthew1027701478
Content-Type: text/plain

matthew		Fri Jul 26 12:37:58 2002 EDT

  Modified files:              
    /loncom/interface	lonsearchcat.pm 
  Log:
  Initial restructuring to use lonmysql.pm.  
  
  Currently, the search is run and then results are returned to the user.  This
  differs from the 'show results as you get them' approach.  Once the search
  interface is changed to allow a second httpd to be used to display the results,
  this functionality will be an improvement.  Trust me.
  
  &output_results is deprecated and is no longer used.  Will be removed in
  the future.  It is to be replaced with &run_search() and &display_results().
  
  &run_search() sends the search query to servers one at a time instead of all
  at once.  Getting hung up on crappy connections is inevitable.  Results are
  stored in the mysql database.
  
  &display_results() querys the mysql database and provides the results to the 
  user.  Sorting of results and searching within results is not yet supported.
  
  A few utility functions have been added:
  
  &write_status will eventually be used to provide the user with information
  on the status of the current search.  Currently used for debugging.
  
  &display_buttons is not used yet but will be made to output 'previous' and
  'next' links like you are used to seeing on search engines.
  
  &copyright_check is used to determine if a search result should be kept.
  This function still requires testing.
  
  &parse_row is used to reconstruct the data structures stored away in the
  MySQL table.  
  
  A large section of data describing the MySQL table used has been added.
  
  
--matthew1027701478
Content-Type: text/plain
Content-Disposition: attachment; filename="matthew-20020726123758.txt"

Index: loncom/interface/lonsearchcat.pm
diff -u loncom/interface/lonsearchcat.pm:1.143 loncom/interface/lonsearchcat.pm:1.144
--- loncom/interface/lonsearchcat.pm:1.143	Tue Jul 16 11:02:06 2002
+++ loncom/interface/lonsearchcat.pm	Fri Jul 26 12:37:58 2002
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # Search Catalog
 #
-# $Id: lonsearchcat.pm,v 1.143 2002/07/16 15:02:06 matthew Exp $
+# $Id: lonsearchcat.pm,v 1.144 2002/07/26 16:37:58 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -87,8 +87,10 @@
 use Apache::File();
 use CGI qw(:standard);
 use Text::Query;
+use DBI;
 use GDBM_File;
 use Apache::loncommon();
+use Apache::lonmysql();
 
 # ---------------------------------------- variables used throughout the module
 
@@ -259,13 +261,11 @@
         $r->print(&search_results_header($searchtype,$pretty_string));
         $r->print("Sending search request to LON-CAPA servers.<br />\n");
         $r->rflush();
-        # Send query statements over the network to be processed by 
-        # either the SQL database or a recursive scheme of 'grep'-like 
-        # actions (for custom metadata).
-        my $reply=&Apache::lonnet::metadata_query($query,$customquery,
-                                               $customshow,$libraries);
+        &run_search($r,$query,$customquery,$customshow,$libraries);
+        &display_results($r,$searchtype,$hidden,$importbutton,
+                         $closebutton);
+
         $r->rflush();
-        &output_results($searchtype,$r,$reply,$hidden);
     } else {
         #
         # We need to get information to search on
@@ -1100,7 +1100,6 @@
             $pretty_search_string .= " with no related words.";
         }
     }
-    &Apache::lonnet::logthis("Search String: $search_string");
     # Build SQL query string based on form page
     my $query='';
     my $concatarg=join(',"    ",',
@@ -1315,6 +1314,400 @@
 ######################################################################
 ######################################################################
 
+=pod
+
+=item &copyright_check()
+
+=cut
+
+######################################################################
+######################################################################
+
+sub copyright_check {
+    my $Metadata = shift;
+    # Check copyright tags and skip results the user cannot use
+    my (undef,undef,$resdom,$resname) = split('/',
+                                              $Metadata->{'url'});
+    # Check for priv
+    if (($Metadata->{'copyright'} eq 'priv') && 
+        (($ENV{'user.name'} ne $resname) &&
+         ($ENV{'user.domain'} ne $resdom))) {
+        return 0;
+    }
+    # Check for domain
+    if (($Metadata->{'copyright'} eq 'domain') &&
+        ($ENV{'user.domain'} ne $resdom)) {
+        return 0;
+    }
+    return 1;
+}
+
+#####################################################################
+#####################################################################
+
+=pod
+
+=item MySQL Table Description
+
+MySQL table creation requires a precise description of the data to be
+stored.  The use of the correct types to hold data is vital to efficient
+storage and quick retrieval of records.  The columns must be described in
+the following format:
+
+=cut
+
+##
+## Restrictions:
+##    columns of type 'text' and 'blob' cannot have defaults.
+##    columns of type 'enum' cannot be used for FULLTEXT.
+##
+my @DataOrder = qw/id title author subject url keywords version notes
+    abstract mime lang owner copyright creationdate lastrevisiondate hostname
+    idx_title idx_author idx_subject idx_abstract idx_mime idx_language 
+    idx_owner idx_copyright/;
+
+my %Datatypes = 
+    ( id        =>{ type         => 'INT',
+                    restrictions => 'NOT NULL',
+                    primary_key  => 'yes',
+                    auto_inc     => 'yes'
+                    },
+      title     =>{ type=>'TEXT'},
+      author    =>{ type=>'TEXT'},
+      subject   =>{ type=>'TEXT'},
+      url       =>{ type=>'TEXT',
+                    restrictions => 'NOT NULL' },
+      keywords  =>{ type=>'TEXT'},
+      version   =>{ type=>'TEXT'},
+      notes     =>{ type=>'TEXT'},
+      abstract  =>{ type=>'TEXT'},
+      mime      =>{ type=>'TEXT'},
+      lang      =>{ type=>'TEXT'},
+      owner     =>{ type=>'TEXT'},
+      copyright =>{ type=>'TEXT'},
+      hostname  =>{ type=>'TEXT'},
+      #--------------------------------------------------
+      creationdate     =>{ type=>'DATETIME'},
+      lastrevisiondate =>{ type=>'DATETIME'},
+      #--------------------------------------------------
+      idx_title     =>{ type=>'FULLTEXT', target=>'title'},
+      idx_author    =>{ type=>'FULLTEXT', target=>'author'},
+      idx_subject   =>{ type=>'FULLTEXT', target=>'subject'},
+      idx_abstract  =>{ type=>'FULLTEXT', target=>'abstract'},
+      idx_mime      =>{ type=>'FULLTEXT', target=>'mime'},
+      idx_language  =>{ type=>'FULLTEXT', target=>'lang'},
+      idx_owner     =>{ type=>'FULLTEXT', target=>'owner'},
+      idx_copyright =>{ type=>'FULLTEXT', target=>'copyright'},
+      );
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &write_status()
+
+=cut
+
+######################################################################
+######################################################################
+sub write_status {
+    my ($r,$string) = @_;
+    $r->print("<pre>".$string."</pre>\n");
+    $r->rflush();
+    return;
+}
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &run_search 
+
+=cut
+
+######################################################################
+######################################################################
+sub run_search {
+    my ($r,$query,$customquery,$customshow,$serverlist) = @_;
+    #
+    my @Servers_to_contact;
+    if (defined($serverlist)) {
+        @Servers_to_contact = @$serverlist;
+    } else {
+        @Servers_to_contact = sort(keys(%Apache::lonnet::libserv));
+    }
+    my %Server_status;
+    #
+    # Timing variables
+    my $starttime = time;
+    my $max_time  = 120;  # seconds for the search to complete
+    #
+    # Create Table
+    #####################################
+    my $table = &Apache::lonmysql::create_table
+        ( { columns => \%Datatypes,
+            column_order => \@DataOrder,
+        } );
+    if (! defined($table)) {
+        # What do I do now?  Print out an error page.
+        &Apache::lonnet::logthis("lonmysql attempted to create a table ".
+                                 "and this was the result:".
+                                 &Apache::lonmysql::get_error());
+        $r->print("An internal error occured with the database.<br />".
+                  "The error has been logged, but you should probably alert".
+                  " your system administrator.");
+        return;
+    }
+    $ENV{'form.table'}=$table;
+    #
+    #####################################
+    my $hitcountsum;
+    my $server; 
+    my $status;
+    while ((time - $starttime < $max_time) && 
+           ((@Servers_to_contact) || keys(%Server_status))) {
+        # Send out a search request if it needs to be done.
+        if (@Servers_to_contact) {
+            # Contact one server
+            my $server = shift(@Servers_to_contact);
+            my $reply=&Apache::lonnet::metadata_query($query,$customquery,
+                                                      $customshow,[$server]);
+            # We should let the user know we have contacted another server
+            ($server) = keys(%$reply);
+            $Server_status{$server} = $reply->{$server};
+            # &write_status($r,"Contacted:$server:reply:".
+            #                   $Server_status{$server});
+            # Hmmm, should we add to $max_time if we contact a server?
+        } else {
+            sleep(1); # wait a sec. to give time for files to be written
+        }
+        while (my ($server,$status) = each(%Server_status)) {
+            if ($status eq 'con_lost') {
+                delete ($Server_status{$server});
+                # &write_status($r,"Removing $server");
+                next;
+            }
+            $status=~/^([\.\w]+)$/; 
+       	    my $datafile=$r->dir_config('lonDaemons').'/tmp/'.$1;
+            if (-e $datafile && ! -e "$datafile.end") {
+                # Let the user know we are receiving data from the server
+                &write_status($r,"$server:Receiving file");
+                next;
+            }
+            if (-e "$datafile.end") {
+                if (-z "$datafile") {
+                    delete($Server_status{$server});
+                    next;
+                }
+                my $fh;
+                if (!($fh=Apache::File->new($datafile))) { 
+                    # Error opening file...
+                    # Tell the user and exit...?
+                    # Should I give up on opening it?
+                    &write_status("Unable to open $datafile");
+                    next;
+                }
+                # Read in the whole file.
+                while (my $result = <$fh>) {
+                    # handle custom fields?  Someday we will!
+                    chomp($result);
+                    next unless $result;
+                    # Parse the result.
+                    my %Fields = &parse_raw_result($result,$server);
+                    $Fields{'hostname'} = $server;
+                    next if (! &copyright_check(\%Fields));
+                    # Store the result in the mysql database
+                    my $result = &Apache::lonmysql::store_row($table,\%Fields);
+                    if (! defined($result)) {
+                        &write_status($r,&Apache::lonmysql::get_error());
+                    }
+                    # &write_status($r,&Apache::lonmysql::get_debug());
+                    $hitcountsum ++;
+                } # End of foreach (@results)
+                $fh->close();
+                # $server is only deleted if the results file has been 
+                # found and (successfully) opened.  This may be a bad idea.
+                delete($Server_status{$server});
+            }
+        }
+        # Finished looping through the servers
+    }
+    &Apache::lonmysql::disconnect_from_db();
+    # We have run out of time or run out of servers to talk to and
+    # results to get.  
+    if ($hitcountsum > 0) {
+        $r->print("<h3>Total results = $hitcountsum</h3></body></html>");
+    }
+    return;
+}
+
+######################################################################
+######################################################################
+=pod
+
+=item &display_buttons
+
+=cut
+
+######################################################################
+######################################################################
+sub display_buttons {
+    my ($low,$high,$otherparms) = @_;
+    my $maxshow = 20;
+    my $lowest = ($low - $maxshow < 0 ? 0 : $low-$maxshow);
+    my $highest = $high + $maxshow;
+    my ($previous,$current,$next);
+    if ($lowest < $low) {
+        $previous = qq{<a href="/adm/searchcat?$otherparms&mode=display&low=$lowest&high=$highest">prev</a>};
+    } else {
+        $previous = "prev";
+    }
+    $current = qq{<a href="/adm/searchcat?$otherparms&mode=display&low=$low&high=$high">reload</a>};
+    $next = qq{<a href="/adm/searchcat?$otherparms&mode=display&low=$high&high=$highest">next</a>};
+    my $result = $previous." ".$current." ".$next;
+    return $result;
+}
+######################################################################
+######################################################################
+
+=pod
+
+=item &display_results
+
+=cut
+
+######################################################################
+######################################################################
+sub display_results {
+    my ($r,$mode,$hidden,$importbutton,$closebutton) = @_;
+    ##
+    ## Set viewing function
+    ##
+    my $viewfunction = $Views{$ENV{'form.viewselect'}};
+    if (!defined($viewfunction)) {
+        $r->print("Internal Error - Bad view selected.\n");
+        $r->rflush();
+        return;
+    }
+    ##
+    ## make query information persistent to allow for subsequent revision
+    ##
+    my $persistent=&make_persistent(\%ENV);
+    ##
+    ## Get the catalog controls setup
+    ##
+    my $action = "/adm/searchcat";
+    if ($mode eq 'Basic') { 
+        $action .= "?reqinterface=basic";
+    } elsif ($mode eq 'Advanced') {
+        $action .= "?reqinterface=advanced";
+    }
+    $r->print(<<CATALOGCONTROLS);
+<form name='results' method="post" action="$action">
+$hidden
+<input type='hidden' name='acts' value='' />
+<input type='button' value='Revise search request'
+onClick='this.form.submit();' />
+$importbutton
+$closebutton
+$persistent
+<hr />
+CATALOGCONTROLS
+    if (! tie(%groupsearch_db,'GDBM_File',$diropendb,&GDBM_WRCREAT,0640)) {
+        $r->print('Unable to tie hash to db file</body></html>');
+        $r->rflush();
+        return;
+    } 
+    #
+    my $fnum;
+    # For now, just query the whole lot and spit them out.
+    my $table = $ENV{'form.table'};
+    my $connection_result = &Apache::lonmysql::connect_to_db();
+    if (!defined($connection_result)) {
+        &write_status($r,&Apache::lonmysql::get_error());
+    }
+    my @Results = &Apache::lonmysql::get_rows($table,'id>=0');
+    #&write_status($r,&Apache::lonmysql::get_debug());
+    #&write_status($r,&Apache::lonmysql::get_error());
+    foreach my $row (@Results) {
+        my %Fields = %{&parse_row(@$row)};
+        my $output="\n<p>\n";
+        if ($ENV{'form.catalogmode'} eq 'interactive') {
+            my $titleesc=$Fields{'title'};
+            $titleesc=~s/\'/\\'/; # '
+            if ($ENV{'form.catalogmode'} eq 'interactive') {
+                $output.=<<END 
+<font size='-1'><INPUT TYPE="button" NAME="returnvalues" VALUE="SELECT"
+onClick="javascript:select_data('$titleesc','$Fields{'url'}')">
+</font>
+<br />
+END
+            }
+        }
+        if ($ENV{'form.catalogmode'} eq 'groupsearch') {
+            $fnum+=0;
+            $groupsearch_db{"pre_${fnum}_link"}=$Fields{'url'};
+            $groupsearch_db{"pre_${fnum}_title"}=$Fields{'title'};
+            $output.=<<END;
+<font size='-1'>
+<input type="checkbox" name="returnvalues" value="SELECT"
+onClick="javascript:queue($fnum)" />
+</font>
+<br />
+END
+# <input type="hidden" name="title$fnum" value="$title" />
+# <input type="hidden" name="url$fnum" value="$url" />
+            $fnum++;
+        }
+        # Render the result into html
+        $output.= &$viewfunction(%Fields);
+        if ($output) {
+            $output.="<hr align='left' width='200' noshade />";
+        }
+        $r->print($output);
+        $r->rflush();
+    }
+    if (@Results < 1) {
+        $r->print("There were no results matching your query");
+    }
+    $r->print("</body></html>");
+    $r->rflush();
+    untie %groupsearch_db;
+    return;
+}
+
+######################################################################
+######################################################################
+
+=pod
+
+=item &parse_row
+
+Parse a row returned from the database.
+
+=cut
+
+######################################################################
+######################################################################
+sub parse_row {
+    my @Row = @_;
+    my %Fields;
+    for (my $i=0;$i<=$#Row;$i++) {
+        $Fields{$DataOrder[$i]}=&Apache::lonnet::unescape($Row[$i]);
+    }
+    $Fields{'language'} = 
+        &Apache::loncommon::languagedescription($Fields{'lang'});
+    $Fields{'copyrighttag'} =
+        &Apache::loncommon::copyrightdescription($Fields{'copyright'});
+    $Fields{'mimetag'} =
+        &Apache::loncommon::filedescription($Fields{'mime'});
+    return \%Fields;
+}
+######################################################################
+######################################################################
+
 =pod 
 
 =item &output_results() 
@@ -1611,6 +2004,7 @@
         $Fields{'title'}='Untitled'; 
     }
     unless ($ENV{'user.adv'}) {
+        # What is this anyway?
         $Fields{'keywords'} = '- not displayed -';
         $Fields{'notes'}    = '- not displayed -';
         $Fields{'abstract'} = '- not displayed -';

--matthew1027701478--