#!/usr/bin/perl
#
# $Name:  $
#
# $Log: importSList.pl,v $
# Revision 1.6  2003/06/07 23:27:11  robert
# Add importing of only pairs of stars.  Needs work.
#
# Revision 1.5  2002/02/20 05:16:05  robc
# Use TASSIV_FITS_PM env variable for
# FITS.pm location
#
# Revision 1.4  2002/02/19 14:56:20  robc
# Add support for AppConfig
# Change to support new db schema
# Add timings of operations
#
# Revision 1.3  2001/12/12 04:32:53  robc
# Add support for keeping track if ID column output from
# DAOPHOT so PSF phomotery may be matched with apeture
# photometry in future scritps.
#
#
# Revision 1.1  2001/11/23 16:11:31  robc
# First version
#

use warnings;
use strict;
use lib( $ENV{ TASSIV_FITS_PM } );

use FITS;
use Getopt::Std;
use IO::Socket;
use DBI;
use AppConfig qw( :argcount :expand );
use Time::HiRes qw( tv_interval gettimeofday );

sub insert_file( $$$$$ );
sub setup_args( $$ );
sub keep_pairs( $$$$ );

autoflush STDOUT;

use constant ID => q$Id: importSList.pl,v 1.6 2003/06/07 23:27:11 robert Exp $;

my $version = join ( ' ', ( split ( ' ', ID ) )[ 1 .. 3 ] );
$version =~ s/,v\b//;
$version =~ s/(\S+)$/($1)/;

my $CONFIG_FILE = $ENV{ TASSIV_CFG }
   || '/tass/src/tassiv_reduce/tassiv_reduce.cfg';
my $c = AppConfig->new(
   {
      CREATE => 1,
      GLOBAL => {
                  EXPAND   => EXPAND_VAR,
                  ARGCOUNT => ARGCOUNT_ONE
      }
   }
   );
$c = setup_args( $c, $CONFIG_FILE );

my $MAX_DIFF     = $c->max_time_diff / 3600 / 24;
my $SITE_CHAR    = "'" . $c->site_char . "'";
my $DB_NAME      = $c->name;
my $DB_HOST      = $c->host;
my $MATCH_RADIUS = $c->match_radius;

#
# Try to connect to the database to use
#
print "Connecting to $DB_NAME at $DB_HOST\n";
my $dbh =
   DBI->connect( "DBI:Pg:dbname=$DB_NAME;host=$DB_HOST", $c->user, $c->passwd,
                 { RaiseError => 1, AutoCommit => 0 } );

my $sth_insert_new_group =
   $dbh->prepare( "INSERT INTO groups( site_id, night ) VALUES( ?, ? )" );

my $sth_select_group_id =
   $dbh->prepare(
      "SELECT group_id FROM groups WHERE site_id = ? AND night = ?" );

my $sth_insert_new_pair =
   $dbh->prepare( "INSERT INTO pairs( pair_id, group_id ) VALUES( ?, ? )" );

my $sth_select_file_name =
   $dbh->prepare( "SELECT name FROM files WHERE name = ?" );

my $sth_insert_files =
   $dbh->prepare( "INSERT INTO files( group_id, pair_id, date, name ) " .
                  "VALUES( ?, ?, ?, ? )" );

my $insert = 
      "INSERT INTO fits( file_id, adc, ra, dec, darktime, exptime ) VALUES( ";
#    "VALUES( ?, '{ " . sprintf( "%s", "?, " x31 ) . " ? }', ?, ?, ?, ? )";
#my $sth_insert_fits =
#    $dbh->prepare( $insert );

my $sth_insert_imported =
   $dbh->prepare( "INSERT INTO imported( file_id ) VALUES( ? )" );

my $sth_update_files =
   $dbh->prepare( "UPDATE files SET ra_min = ?, ra_max = ?,"
                  . " dec_min = ?, dec_max = ? WHERE file_id = ?" );

my $sth_just_imported =
   $dbh->prepare( "SELECT obs_id, ra, dec FROM obs_i NATURAL JOIN files"
                  . " WHERE group_id = ? AND pair_id = ?" );

my $sth_match_imported =
   $dbh->prepare( qq/
SELECT obs_id, ra, dec
FROM obs_v NATURAL JOIN files
WHERE  group_id = ?
   AND pair_id  = ?
   AND circle_contain_pt(
         circle( point( ra, dec ), $MATCH_RADIUS ),
         point( ?, ? ) )
   AND ra  BETWEEN (? - $MATCH_RADIUS)::real AND (? + $MATCH_RADIUS)::real
   AND dec BETWEEN (? - $MATCH_RADIUS)::real AND (? + $MATCH_RADIUS)::real
/);

my $values = $c->db_columns;
$values =~ s/\w+/?/g;

my ( @line, $data, $per, @file_id, $index, $count, $t0, $elapsed );
my $total = 0;
my $running_total = 0;
my $kept = 0;
my $kept_small = 0;
my $running_kept = 0;
my $running_kept_small = 0;
my $vacuum = 0;

while ( my $dir = shift )
   {
   $total = 0;
   $running_total = 0;
   $kept = 0;
   $kept_small = 0;
   $running_kept = 0;
   $vacuum = 0;

   print "Finding motching files for $dir\n";
   my $matched_ref = FITS_find_matched( $dir, $MAX_DIFF );

   my @files = sort( keys %$matched_ref );

   my( $night_id, $group_id );
   $files[0] =~ /..r.(\d{4}).*\.\w+/;
   $night_id = $1;
   $sth_select_group_id->execute( 'k', $night_id );

   if ( 0 == $sth_select_group_id->rows )
      {
      $sth_insert_new_group->execute( 'k', $night_id );
      $sth_insert_new_group->finish;
      $sth_select_group_id->finish;
      $sth_select_group_id->execute( 'k', $night_id );
      $group_id = $sth_select_group_id->fetchrow_array;
      print "Using new group_id $group_id\n";
      }
   else
      {
      $group_id = $sth_select_group_id->fetchrow_array;
      print "Using existing group_id $group_id\n";
      }
   $sth_select_group_id->finish;

   for ( @files )
      {
      next unless -e "$_.apm" && -e "$matched_ref->{ $_ }.apm";

      my $full_file_1   = $_;
      my $full_file_2   = $matched_ref->{ $_ };
      my $import_file_1 = "$full_file_1.apm";
      my $import_file_2 = "$full_file_2.apm";
      my $file_name_1   = ( split '/', $full_file_1 )[ -1 ];
      my $file_name_2   = ( split '/', $full_file_2 )[ -1 ];

      print $import_file_1;

      $sth_select_file_name->execute( $file_name_1 );
      if ( 0 != $sth_select_file_name->rows )
         {
         print ", $import_file_1 already imported\n";
         $sth_select_file_name->finish;
         next;
         }
      $sth_select_file_name->finish;

      my $pair_id =
         $dbh->selectrow_array( "SELECT nextval('pairs_pair_id_seq')" );

      $sth_insert_new_pair->execute( $pair_id, $group_id );
      $sth_insert_new_pair->finish;

      print "... ";
      my $i1 = insert_file( $group_id, $pair_id,
                             $full_file_1, $import_file_1, $file_name_1 );

      print "$import_file_2... ";
      my $i2 = insert_file( $group_id, $pair_id,
                             $full_file_2, $import_file_2, $file_name_2 );
      my $t1 = keep_pairs( $group_id, $pair_id, $i1, $i2 );
      $dbh->commit;
      $total += $i1 + $i2;
      $kept += $t1;
      $kept_small += $i1 < $i2 ? $i1 : $i2;
      $vacuum += $t1;
      if ( $vacuum > 1000 )
         {
         $vacuum = 0;
#         &vacuum_temp( );
         }
      }
   next if !$total;
   printf "%d kept pairs out of %d total entries (%.1f%%)\n",
          $kept, $total, 100 * $kept / $kept_small;
   $running_kept += $kept;
   $running_total += $total;
   $running_kept_small += $kept_small;
   }

if ( $running_total )
   {
   printf "%d kept pairs out of %d total entries (%.1f%%)\n",
          $running_kept, $running_total,
          100 * $running_kept / $running_kept_small;
   }

&vacuum( 'FULL' );
exit;

sub vacuum_temp
   {
   my $t0;
   print "Vacuuming...:\n";
   local $dbh->{ AutoCommit } = 1;
   for ( qw/groups pairs files temp_obs_u temp_obs_b
            temp_obs_v temp_obs_r temp_obs_i obs_u obs_b
            obs_v obs_r obs_i/ )
      {
      printf "\t %20s - ", $_;
      $t0 = [ gettimeofday ];
      $dbh->do( "VACUUM ANALYZE $_" );
      $elapsed = tv_interval( $t0, [ gettimeofday ] );
      printf "%.0f seconds\n", $elapsed;
      }
   }

sub vacuum
   {
   my $param = shift || '';
   my $t0;

   print "Vacuuming...:\n";
   local $dbh->{ AutoCommit } = 1;
   for ( qw/groups pairs files fits temp_obs_u temp_obs_b
            temp_obs_v temp_obs_r temp_obs_i/ )
      {
      printf "\t %20s - ", $_;
      $t0 = [ gettimeofday ];
      $dbh->do( "VACUUM $param ANALYZE $_" );
      $elapsed = tv_interval( $t0, [ gettimeofday ] );
      printf "%.0f seconds\n", $elapsed;
      }
   }

sub insert_file( $$$$$ )
   {
   my ( $group_id, $pair_id, $full_file, $i_file, $f_name ) = @_;

   my $date = FITS_get_date_time( $full_file );
   my $band = uc eval FITS_get_key_value( $full_file, 'FILTER' );

   my @adc = map { FITS_get_key_value( $full_file,
                                       sprintf "ADC_%02d", $_ ) } 00..31;
   my $ra = FITS_get_key_value( $full_file, "RA" );
   my $dec = FITS_get_key_value( $full_file, "DEC" );
   my $darktime = FITS_get_key_value( $full_file, "DARKTIME" );
   my $exptime = FITS_get_key_value( $full_file, "EXPTIME" );

   my ( $size_x, $size_y )  = FITS_get_size( $full_file );
   my ( $ra_cov, $dec_cov ) = FITS_get_pixel_size( $full_file );
   $ra_cov  = abs( $ra_cov * $size_x );
   $dec_cov = abs( $dec_cov * $size_y );
   my $t0 = [ gettimeofday ];

   $sth_insert_files->execute( $group_id, $pair_id,
                               "$date->[0] $date->[1]", $f_name );
   my $file_id = $dbh->selectrow_array( "SELECT currval('files_file_id_seq')" );
   $dbh->do( $insert . "$file_id, '{" .
             join( ", ", @adc ) . "}', $ra, $dec, $darktime, $exptime )" );

   $sth_insert_imported->execute( $file_id );
   my $sth_insert_obs =
      $dbh->prepare( "INSERT INTO temp_obs_$band ( file_id, "
                     . join( ', ', $c->db_columns )
                     . " ) VALUES ( ?, $values )" );

   open( FILE, $i_file );

   if ( $c->copy == 1 )
      {
      $index = $dbh->selectrow_array( "SELECT nextval('obs_id_seq')" );
      $dbh->do( "COPY temp_obs_$band FROM stdin" );
#      $dbh->do( "COPY obs_$band FROM stdin" );
      }

   $count = 0;
   while ( <FILE> )
      {
      my $temp = $c->db_values;
      my @data = ( split )[ map { split /,?\s+/ } @$temp ];

      if ( $c->copy == 1 )
         {
         push @data, $index, $file_id, 'FALSE', 'FALSE', '\N', '\N';
         $dbh->func( join ( "\t", @data ) . "\n", 'putline' );
         }
      else
         {
         unshift @data, $file_id;
         $sth_insert_obs->execute( @data );
         }
      ++$index;
      ++$count;
      }
   close FILE;

   if ( $c->copy == 1 )
      {
      $dbh->func( "\\.\n", 'putline' );
      $dbh->func( 'endcopy' );
      $dbh->do( "SELECT setval('obs_id_seq',$index)" );
      }

   my @range =
      $dbh->selectrow_array(
      "SELECT min( ra ), max( ra ), min( dec ), max( dec )"
#      . " FROM obs_$band WHERE file_id = $file_id" );
      . " FROM temp_obs_$band WHERE file_id = $file_id" );

   #
   # If the difference is too large, we need to swap the ra values (it wrapped)
   if ( ( $range[ 1 ] - $range[ 0 ] ) > ( 10 * $ra_cov ) )
      {
      unshift @range, splice @range, 1, 1;
      }
   $sth_update_files->execute( @range, $file_id );

   $elapsed = tv_interval( $t0, [ gettimeofday ] );
   printf "%5d stars imported %4.0f per second\n", $count, $count / $elapsed;

   return $count;
   }

sub setup_args( $$ )
   {
   my ( $c, $file ) = @_;
   $c->define(
               Merge_Stars_match_radius => {
                                             DEFAULT => .002,
                                             ALIAS   => 'match_radius'
               },
               Import_SList_max_time_diff => {
                                               DEFAULT => 3,
                                               ALIAS   => 'max_time_diff'
               },
               DB_name => {
                            DEFAULT => 'tassiv',
                            ALIAS   => 'name'
               },
               DB_user => {
                            DEFAULT => 'robert',
                            ALIAS   => 'user'
               },
               DB_passwd => {
                            DEFAULT => '',
                            ALIAS   => 'passwd'
               },
               DB_host => {
                            DEFAULT => 'localhost',
                            ALIAS   => 'host'
               }
               );
   $c->define( 'Import_SList_site_char|site_char=s' );
   $c->define( 'Import_SList_db_columns|db_columns=s' );
   $c->define( 'Import_SList_db_values|db_values=i@' );
   $c->define( 'Import_SList_copy|copy|c!' );
   $c->file( $file );

   $c->args( \@ARGV );

   return $c;
   }

sub keep_pairs( $$$$ )
   {
   my( $group_id, $pair_id, $i1, $i2 ) = @_;    

   my $small = $i1 < $i2 ? $i1 : $i2;
   my $kept = 0;

   my $t0 = [ gettimeofday ];

if ( 0 )
   {
   $sth_just_imported->execute( $group_id, $pair_id );
   my %new_select;
   $sth_just_imported->bind_columns(
                    \( @new_select{ @{ $sth_just_imported->{ NAME_lc } } } ) );

   $dbh->do( 'CREATE TEMP TABLE keep( obs_id integer )' );

   while ( $sth_just_imported->fetch )
      {
      $sth_match_imported->execute( $group_id, $pair_id,
                                    $new_select{ ra }, $new_select{ dec },
                                    $new_select{ ra }, $new_select{ ra },
                                    $new_select{ dec }, $new_select{ dec } );
      if ( $sth_match_imported->rows != 1 )
         {
         $dbh->do( "DELETE FROM obs_i WHERE obs_id = $new_select{ obs_id }" );
         }
      else
         {
         my @data = $sth_match_imported->fetchrow_array;
         $dbh->do( "INSERT INTO keep VALUES( $data[0] )" );
         ++$kept;
         }
      $sth_match_imported->finish;
      }
   $sth_just_imported->finish;
   $dbh->do( 'DELETE FROM obs_v WHERE obs_id NOT IN (SELECT obs_id FROM keep)' );
   $dbh->do( 'DROP TABLE keep' );
   }
elsif( 1 )
   {
   $dbh->do( qq/
SELECT i.obs_id AS i_obs_id,
       v.obs_id AS v_obs_id
INTO TEMP obs_keep
FROM (temp_obs_v NATURAL JOIN files) AS v,
   (SELECT obs_id, file_id, ra, dec
    FROM temp_obs_i NATURAL JOIN files
    WHERE  group_id = $group_id
       AND pair_id  = $pair_id) AS i
WHERE  v.group_id = $group_id
   AND v.pair_id = $pair_id
   AND circle_contain_pt(
         circle( point( i.ra, i.dec ), $MATCH_RADIUS ),
         point( v.ra, v.dec ) )
   AND v.ra  BETWEEN (i.ra - $MATCH_RADIUS)::real
                 AND (i.ra + $MATCH_RADIUS)::real
   AND v.dec BETWEEN (i.dec - $MATCH_RADIUS)::real
                 AND (i.dec + $MATCH_RADIUS)::real
/ );
print ".";

   $dbh->do(
   "CREATE INDEX obs_keep_v_index ON obs_keep USING btree( v_obs_id )" );
   $dbh->do(
   "CREATE INDEX obs_keep_i_index ON obs_keep USING btree( i_obs_id )" );
print ".";

   $dbh->do( qq/
INSERT INTO obs_v
SELECT *
FROM temp_obs_v
WHERE obs_id IN (SELECT v_obs_id FROM obs_keep)
/ );
print ".";
   $dbh->do( qq/
INSERT INTO obs_i
SELECT *
FROM temp_obs_i
WHERE obs_id IN (SELECT i_obs_id FROM obs_keep)
/ );
print ".";

   $kept += $dbh->selectrow_array( "SELECT COUNT(*) FROM obs_keep" );
   $dbh->do( 'DROP TABLE obs_keep' );
   $dbh->do( qq/
DELETE FROM temp_obs_i
WHERE file_id IN
   (SELECT file_id
    FROM temp_obs_i NATURAL JOIN files
    WHERE  group_id = $group_id
       AND pair_id  = $pair_id)
/ );
   $dbh->do( qq/
DELETE FROM temp_obs_v
WHERE file_id IN
   (SELECT file_id
    FROM temp_obs_v NATURAL JOIN files
    WHERE  group_id = $group_id
       AND pair_id  = $pair_id)
/ );
   }
else
   {
   $dbh->do( qq/
INSERT INTO obs_v
SELECT *
FROM temp_obs_v
/ );
print ".";
   $dbh->do( qq/
INSERT INTO obs_i
SELECT *
FROM temp_obs_i
/ );
print ".";

   $kept = 1;
   }
   

   my $elapsed = tv_interval( $t0, [ gettimeofday ] );
   printf "Kept %4d (%4.1f%%) pairs of stars at %5.1f per second (%5.1f)\n",
          $kept, 100 * $kept / $small, $kept / $elapsed, $elapsed;

   return $kept;
   }
