#!/usr/bin/perl
#
# $Name:  $
#
# $Log: mergeStars.pl,v $
# Revision 1.6  2003/06/07 23:54:50  robert
# Use imported list to limit stars required to match.
#
# Revision 1.5  2002/02/20 14:08:30  robc
# Correct a vacuum statement
#
# Revision 1.4  2002/02/20 05:17:56  robc
# Use TASSIV_FITS_PM env variable for FITS.pm location.
# Add back observation update for new stars.
#
# Revision 1.3  2002/02/19 14:58:01  robc
# Add support for AppConfig
# Change to support new db schema
# Add timings of operations
#
# Revision 1.2  2001/12/11 06:01:09  robc
# Add CVS header data and log from last commit
#
# Revision 1.1  2001/11/23 16:11:29  robc
# First version
#

use warnings;
use strict;
use IO;
use DBI;
use AppConfig qw( :argcount :expand );
use Time::HiRes qw( tv_interval gettimeofday );

sub setup_args( $$ );

STDOUT->autoflush;

use constant ID => q$Id: mergeStars.pl,v 1.6 2003/06/07 23:54:50 robert Exp $;

my $version = join ( ' ', ( split ( ' ', ID ) )[ 1 .. 3 ] );
$version =~ s/,v\b//;
$version =~ s/(\S+)$/($1)/;

my $CONFIG_FILE = $ENV{ TASSIV_CFG }
   || '/tass/src/tassiv_reduce/tassiv_reduce.cfg';
my $c = AppConfig->new(
   {
      CREATE => 1,
      GLOBAL => {
                  EXPAND   => EXPAND_VAR,
                  ARGCOUNT => ARGCOUNT_ONE
      }
   }
   );
$c = setup_args( $c, $CONFIG_FILE );

my $DB_NAME      = $c->name;
my $DB_HOST      = $c->host;
my $MATCH_RADIUS = $c->match_radius;
my $MAX_SMAG     = $c->max_smag;
my $MIN_STARS    = $c->min_stars;
my $VACUUM       = $c->vacuum;

my $dbh =
   DBI->connect( "DBI:Pg:dbname=$DB_NAME;host=$DB_HOST", "", "",
                 { RaiseError => 1, AutoCommit => 0 } );

sub done;
$SIG{ INT } = \&done;

my $vacuum = <<__VACUUM__;

VACUUM ANALYZE new_reference_loc;
VACUUM ANALYZE catalog;
VACUUM ANALYZE obs_u;
VACUUM ANALYZE obs_b;
VACUUM ANALYZE obs_v;
VACUUM ANALYZE obs_r;
VACUUM ANALYZE obs_i;
__VACUUM__

#
# This one looks for stars in the observations which haven't matched yet but
# are within the radius of 'this' star
my $sql = <<__END_FIND_MATCHING_STARS__;

SELECT ra, dec, obs_id, 'XXX' AS band
FROM obs_XXX
WHERE circle_contain_pt(
         circle( point( ?, ? ), $MATCH_RADIUS ),
         point( ra, dec ) )
   AND ra  BETWEEN (? - $MATCH_RADIUS)::real AND (? + $MATCH_RADIUS)::real
   AND dec BETWEEN (? - $MATCH_RADIUS)::real AND (? + $MATCH_RADIUS)::real
   AND star_id ISNULL
   AND smag < $MAX_SMAG
__END_FIND_MATCHING_STARS__

my $comp_sql = join 'UNION',
   map { my $x = $sql; $x =~ s/XXX/$_/g; $x } ( 'v', 'i' );
#   map { my $x = $sql; $x =~ s/XXX/$_/g; $x } ( 'u', 'b', 'v', 'r', 'i' );
my $sth_find_matching_stars = $dbh->prepare( $comp_sql );

#
# This one looks into the frozen stars and reference to make sure the 'current'
# star doesn't match an existing one
$sql = <<__END_FIND_EXISTING_STARS__;

SELECT star_id, ra, dec
FROM reference_ubvri
WHERE circle_contain_pt(
         circle( point( ra, dec ), $MATCH_RADIUS ),
         point( ?, ? ) )
   AND ra  BETWEEN (? - $MATCH_RADIUS)::real AND (? + $MATCH_RADIUS)::real
   AND dec BETWEEN (? - $MATCH_RADIUS)::real AND (? + $MATCH_RADIUS)::real
UNION
SELECT star_id, ra, dec
FROM new_reference_loc
WHERE circle_contain_pt(
         circle( point( ra, dec ), $MATCH_RADIUS ),
         point( ?, ? ) )
   AND ra  BETWEEN (? - $MATCH_RADIUS)::real AND (? + $MATCH_RADIUS)::real
   AND dec BETWEEN (? - $MATCH_RADIUS)::real AND (? + $MATCH_RADIUS)::real
__END_FIND_EXISTING_STARS__
my $sth_find_existing_stars = $dbh->prepare( $sql );

#
# Set up an insert now ...
my $sth_insert_catalog =
   $dbh->prepare( "INSERT INTO catalog( ra, dec ) VALUES( ?, ? )" );

#
# and a select...
my $sth_star_id_select = $dbh->prepare( "SELECT currval('star_id_seq')" );

#
# and another select...
my $sth_ra_dec_select =
   $dbh->prepare(
              "SELECT star_id, ra, dec, count FROM catalog WHERE star_id = ?" );

#
# and another insert...
my $sth_new_reference_loc_insert =
   $dbh->prepare(
   "INSERT INTO new_reference_loc( star_id, ra, dec ) " . "VALUES( ?, ?, ? )" );

#
# and an update
sub observation_update( $$$ )
   {
   my ( $band, $star_id, $obs_id ) = @_;

   my $sth_update =
      $dbh->prepare_cached(
                          "UPDATE obs_$band SET star_id = ? WHERE obs_id = ?" );
   $sth_update->execute( $star_id, $obs_id );
   }

#
# and a update
my $sth_welch_index_update =
   $dbh->prepare( "UPDATE welch_index SET updated = FALSE WHERE star_id = ?" );

#
# First off, lets find all the stars which have no star_id and have been
# recently imported...
$sql = <<__END_FIND_NULL_STARS__;

SELECT ra, dec, obs_id, 'XXX'
FROM obs_XXX AS band, (SELECT file_id FROM temp_imported) AS imp
WHERE  star_id ISNULL
   AND band.file_id = imp.file_id
   AND smag < $MAX_SMAG
__END_FIND_NULL_STARS__

$comp_sql = join 'UNION',
   map { my $x = $sql; $x =~ s/XXX/$_/g; $x } ( 'v', 'i' );
#   map { my $x = $sql; $x =~ s/XXX/$_/g; $x } ( 'u', 'b', 'v', 'r', 'i' );
my $sth_null_stars = $dbh->prepare( $comp_sql );

#
# And we'll create a table to keep the file_id's we used
$dbh->do(
   'SELECT * INTO TEMP temp_imported FROM (SELECT * FROM imported) AS i' );

print "Finding unassigned stars: ";
$sth_null_stars->execute;

#
#  With the following bind, after each fetch, we'll end up with a hash of
#  the columns from the select.  If we add to the select, we don't have
#  to change the bind either...
my %null_select;
$sth_null_stars->bind_columns(
                       \( @null_select{ @{ $sth_null_stars->{ NAME_lc } } } ) );

printf "found %d of them\n", $sth_null_stars->rows;

#
# This hash will keep track of obs_id's which have already been assigned
# to a star in this session, thus reducing the number of queries we need
my %seen;
my $new          = 0;
my $new_total    = 0;
my $update       = 0;
my $update_total = 0;
my $many         = 0;
my $not_enough   = 0;
my $count        = 0;
my $stats        = 0;
my $last_vacuum  = 0;

#
# Timing related stuff
my $here = 0;
my $t0   = [ gettimeofday ];
my $elapsed;

while ( $sth_null_stars->fetch )
   {

   #
   # If we've seen it, remove it (obs_id's are unique, so we'll never see
   # it again), and continue with the next one
   if ( exists $seen{ $null_select{ obs_id } } )
      {
      delete $seen{ $null_select{ obs_id } };
      next;
      }

   #
   # Check to see if it's time to vacuum...
   if ( ( ($new_total + $update_total) - $last_vacuum ) > $VACUUM )
      {
      $elapsed = tv_interval( $t0, [ gettimeofday ] );
      $here    = 0;
      $t0      = [ gettimeofday ];
      $last_vacuum = $new_total + $update_total;
#      $dbh->do( $vacuum );
#      vacuum( );
      }

   my $ra  = $null_select{ ra };
   my $dec = $null_select{ dec };

   #
   # If this star matches 1 existing one (from the new_reference_loc table),
   # then we can add this star to it's influence.  While we're at it,
   # lets go ahead and add all appropriate stars to it's influence
   $sth_find_existing_stars->execute(
                                      $ra, $dec, $ra, $ra, $dec, $dec,
                                      $ra, $dec, $ra, $ra, $dec, $dec
                                      );

   if ( $sth_find_existing_stars->rows == 1 )
      {
      ++$update;

      #
      # Grab the id of what we're matching to
      my ( $star_id, $lra, $ldec ) = $sth_find_existing_stars->fetchrow_array;

      #
      # Since we're adding a star, update the welch index so it'll be
      # calculated again
      $sth_welch_index_update->execute( $star_id );

      #
      # Here, we'll find all the stars to update
      my @m_data = ( $lra, $ldec, $lra, $lra, $ldec, $ldec );
      $sth_find_matching_stars->execute( map { @m_data } ( 1 .. 2 ) );
#      $sth_find_matching_stars->execute( map { @m_data } ( 1 .. 5 ) );

      my %matching_select;
      $sth_find_matching_stars->bind_columns(
          \( @matching_select{ @{ $sth_find_matching_stars->{ NAME_lc } } } ) );

      #
      # Now, go update all the stars which match.  This will fire the
      # trigger in the database to actually do the work of bringing the
      # data from the observations table into the catalog table
      while ( $sth_find_matching_stars->fetch )
         {

         #
         # If we've seen it, remove it (obs_id's are unique, so we'll never see
         # it again), and continue with the next one
         if ( exists $seen{ $matching_select{ obs_id } } )
            {
            delete $seen{ $matching_select{ obs_id } };
            next;
            }
         $seen{ $matching_select{ obs_id } } = 1;
         ++$count;
         ++$update_total;
         observation_update(
                             $matching_select{ band }, $star_id,
                             $matching_select{ obs_id }
                             );
         if ( !( ++$here % ( $VACUUM / 10 ) ) )
            {
            $elapsed = tv_interval( $t0, [ gettimeofday ] );
            printf "%6.1f star operations/second n=%d(%d) u=%d(%d) x=%d m=%d\n",
               ( $VACUUM / 10 ) / $elapsed, $new, $new_total, $update,
               $update_total, $not_enough, $many;
            $t0 = [ gettimeofday ];
            }
         }
      $dbh->commit;
      next;
      }
   #
   # If it matches more than one, then we don't update anything (yet)
   elsif ( $sth_find_existing_stars->rows > 1 )
      {
      $seen{ $null_select{ obs_id } } = 1;
      ++$count;
      ++$many;
      if ( !( ++$here % ( $VACUUM / 10 ) ) )
         {
         $elapsed = tv_interval( $t0, [ gettimeofday ] );
         printf "%6.1f star operations/second n=%d(%d) u=%d(%d) x=%d m=%d\n",
            ( $VACUUM / 10 ) / $elapsed, $new, $new_total, $update,
            $update_total, $not_enough, $many;
         $t0 = [ gettimeofday ];
         }
      next;
      }

   #
   # Here, we matched no existing stars, so we'll see if we can create
   # a new one, given we have enough stars matched.  Note that this is
   # total stars matching in all bands.
   my @m_data = ( $ra, $dec, $ra, $ra, $dec, $dec );
   $sth_find_matching_stars->execute( map { @m_data } ( 1 .. 2 ) );
   my %matching_select;
   $sth_find_matching_stars->bind_columns(
       \( @matching_select{ @{ $sth_find_matching_stars->{ NAME_lc } } } ) );
#   $sth_find_matching_stars->execute( map { @m_data } ( 1 .. 5 ) );
   if ( $sth_find_matching_stars->rows >= $MIN_STARS )
      {
      ++$new;

      #
      # Here, we create the entries in the catalog table so the update
      # trigger can do it's work
      $sth_insert_catalog->execute( $ra, $dec );
      $sth_star_id_select->execute;
      my $star_id = $sth_star_id_select->fetchrow_array;

      #
      # Now, go update all the stars which match.  This will fire the
      # trigger in the database to actually do the work of bringing the
      # data from the observations table into the catalog table
      # At least in theory.  Currently, the trigger checks the use column
      # in the obs_X tables before adding to the catalog table
      while ( $sth_find_matching_stars->fetch )
         {

         #
         # If we've seen it, remove it (obs_id's are unique, so we'll never see
         # it again), and continue with the next one
         if ( exists $seen{ $matching_select{ obs_id } } )
            {
            delete $seen{ $matching_select{ obs_id } };
            next;
            }
         $seen{ $matching_select{ obs_id } } = 1;
         ++$count;
         ++$new_total;

         observation_update(
                             $matching_select{ band }, $star_id,
                             $matching_select{ obs_id }
                             );
         if ( !( ++$here % ( $VACUUM / 10 ) ) )
            {
            $elapsed = tv_interval( $t0, [ gettimeofday ] );
            printf "%6.1f star operations/second n=%d(%d) u=%d(%d) x=%d m=%d\n",
               ( $VACUUM / 10 ) / $elapsed, $new, $new_total, $update,
               $update_total, $not_enough, $many;
            $t0 = [ gettimeofday ];
            }
         }

      #      $sth_ra_dec_select->execute( $star_id );
      #      my @frozen_loc = $sth_ra_dec_select->fetchrow_array();
      #      $sth_frozen_loc_insert->execute( @frozen_loc );
      $dbh->commit;
      }
   else
      {
      #
      # We'll mark all the stars we've found as no-go (add to the hash)
      $seen{ $null_select{ obs_id } } = 1;
      ++$count;
      ++$not_enough;
      while ( $sth_find_matching_stars->fetch )
         {
         #
         # If we've seen it, remove it (obs_id's are unique, so we'll never see
         # it again), and continue with the next one
         if ( exists $seen{ $matching_select{ obs_id } } )
            {
            delete $seen{ $matching_select{ obs_id } };
            next;
            }
         $seen{ $matching_select{ obs_id } } = 1;
         ++$count;
         ++$not_enough;
         }

      if ( !( ++$here % ( $VACUUM / 10 ) ) )
         {
         $elapsed = tv_interval( $t0, [ gettimeofday ] );
         printf "%6.1f star operations/second n=%d(%d) u=%d(%d) x=%d m=%d\n",
            ( $VACUUM / 10 ) / $elapsed, $new, $new_total, $update,
            $update_total, $not_enough, $many;
         $t0 = [ gettimeofday ];
         }
      }
   }

$dbh->do(
   'DELETE FROM imported WHERE file_id IN (SELECT * FROM temp_imported)' );
$dbh->commit;

vacuum( );
done();

exit;

sub vacuum
   {
   my $param = shift || '';
   print "Vacuuming...:\n";
   local $dbh->{ AutoCommit } = 1;
   for ( qw/new_reference_loc catalog obs_u obs_b obs_v obs_r obs_i/ )
      {
      printf "\t %20s - ", $_;
      $t0 = [ gettimeofday ];
      $dbh->do( "VACUUM $param ANALYZE $_" );
      $elapsed = tv_interval( $t0, [ gettimeofday ] );
      printf "%.0f seconds\n", $elapsed;
      }
   $t0 = [ gettimeofday ];
   }

sub done
   {
   $sth_find_matching_stars->finish;
   $sth_find_existing_stars->finish;
   $sth_star_id_select->finish;
   $sth_ra_dec_select->finish;
   $sth_null_stars->finish;
   $dbh->disconnect;

   print "\nCreated $new new stars from $new_total\n";
   print
"Didn't use $not_enough stars because there were't enough to make new ones\n";
   print "Updated $update stars from $update_total\n";
   print "Couldn't update $many stars because they matched more than one\n";
   exit;
};

sub setup_args( $$ )
   {
   my ( $c, $file ) = @_;

   $c->define(
               Merge_Stars_match_radius => {
                                             DEFAULT => .002,
                                             ALIAS   => 'match_radius'
               },
               Merge_Stars_max_smag => {
                                         DEFAULT => .1,
                                         ALIAS   => 'max_smag'
               },
               Merge_Stars_vacuum => {
                                       DEFAULT => 10000,
                                       ALIAS   => 'vacuum'
               },
               Merge_Stars_min_stars => {
                                          DEFAULT => 11,
                                          ALIAS   => 'min_stars'
               },
               DB_name => {
                            DEFAULT => 'tassiv',
                            ALIAS   => 'name'
               },
               DB_host => {
                            DEFAULT => 'localhost',
                            ALIAS   => 'host'
               }
               );
   $c->file( $file );

   $c->args( \@ARGV );

   return $c;
   }
