# multipointMap.pl - build composite RH/Linkage maps with CarthaGene
#   requires carthaPerl.pl inline interface to CarthaGene
#
# Usage:  perl multipointMap.pl \
#              -l <linkage dir> \
#              -r <rh1 dir, rh2 dir,...> 
#              -d <result dir> \
#              -c <chromosome #> \
#		[ -p -t | -n ]
# 
# Input directories and files must be set up before running multipointMap.
# The linkage and each RH directory must contain a subdirectory for the 
# specified chromosome.  The chromosome-specific linkage subdirectory must 
# contain the file "vector.lm", with the linkage data for that chromosome
# formatted for CarthaGene (translate CRIMAP chrompics).  
#
# Each chromosome-specific RH directory must contain the files 
#     "vectors.rhh" (haploid RH data) 
#     "vectors.rhd" (diploid RH data)
#     "vectors.bc" (backcross RH data)  
# Multiple RH data sets can be included - each must have its own 
# directory structure.  Markers common to two or more data sets (linkage or RH)
# must have the same name in each data set.
#
# Results can be stored in a database using the perl DBI, but some code 
# will need to be uncommented and additional perl scripts are needed.
#
# This code is the marker ordering routine from a set of scripts developed
# to construct composite maps of the bovine genome.  It can be used 
# stand-alone, but is intended to be used with the complete set of scripts.
#
# Contact snelling at email.marc.usda.gov 
# with any bug reports, questions, comments, suggestions, ... 
#

use strict;
use Getopt::Long;

do '/PATH/TO/carthaPerl.pl'; #set path to carthaPerl.pl 
#do '/PATH/TO/ConnectPg.pl';  #set path to ConnectPg.pl and pgSetup.pl
#do '/PATH/TO/pgSetup.pl';    #   to store maps in a Postgresql database
#                                  ConnectPg.pl has connectPg() subroutine
#                                      for establishing connection
#                                  pgSetup.pl defines the global %postgres
#                                      hash with setPg().  %postgres includes
#                                      database user and password, server,
#                                      port, and db name

#undef our %postgres;
#%postgres = setPg();

our $CARTA_EM_BIN = "/home/snelling/CarthaGene-devel/CarthaGene/grid/cartaEM";
chomp ( our $RUN_ON_GRID = `which qsub` );
if( ! defined $RUN_ON_GRID ){
  undef $CARTA_EM_BIN;
}

our $checkpointFrequency = 50;
our $mergeBy = 'gen';

my %doMaps = parseArgs();
my $iMapDir = $doMaps{dir};
chomp( my $startDir = `pwd` );
delete $doMaps{dir};

my $mapTable = $doMaps{table};
delete $doMaps{table};

my $mapList = $doMaps{maps};
delete $doMaps{maps};

foreach my $m ( keys %doMaps ){
  chdir "$iMapDir/$m";
  $mergeBy = 'order';
  buildMap( [@{$doMaps{$m}}], "rhh", $mapList, $mapTable ); # initial order
                                                            # with rh haploid data
  buildMap( [@{$doMaps{$m}}], "rhd", $mapList, $mapTable ); # compute maps/likelihood
                                                            # with rh diploid data
  $mergeBy = 'gen';
  buildMap( [@{$doMaps{$m}}], "bc", $mapList, $mapTable );  # project onto common scale
                                                            # with rh formatted s backcross
  chdir $startDir;
}

sub usage {
  die "multipointMap.pl -r \"rh map1 dir,map2 dir,...\"
         -l linkage map dir
         -c \"chromosome1,chromosome2,...\"
       [ -p ]
       [ -f checkpoint frequency]
       [ -d result directory ]\n";
}

sub parseArgs {
  chomp( my $cd = `pwd` );

  undef my $rhmapsS;
  undef my $linkmap;
  undef my $chromS;
  undef my $purge;
  undef my $noStore;
  my $mapTable = "imap_assignments_bos";
  
  my $iMapDir = "integrated";
  my $result = GetOptions ( "rhmaps=s" => \$rhmapsS,
			    "linkage=s" => \$linkmap,
			    "chromosomes=s" => \$chromS,
			    "directory=s" => \$iMapDir,
			    "purge-checkpoint-maps" => \$purge,
			    "frequency=i" => \$checkpointFrequency,
			    "table=s" => \$mapTable,
			    "no_database_storage" => \$noStore
			  );

  if( !defined $rhmapsS || !defined $chromS || !defined $linkmap ){
    usage();
  }

  if( -e $iMapDir && ! -d $iMapDir ){
    die "$iMapDir is not a directory\n";
  }
  if( ! -d $iMapDir ){
    mkdir $iMapDir;
  }
  if( $RUN_ON_GRID ){
    if( ! -d "$iMapDir/$chromS/grid" ){
      mkdir "$iMapDir/$chromS/grid";
    }
    if( ! -d "$iMapDir/$chromS/grid/scripts" ){
      mkdir "$iMapDir/$chromS/grid/scripts";
    }
  }

  if( $noStore ){
    undef $mapTable;
  }

  #$rhmapsS =~ tr/[A-Z]/[a-z]/;
  #$linkmap =~ s/marc/linkage/;

  my @rhmaps = split( /[\s+,]/, $rhmapsS );
  my @chroms = split( /[\s+,]/, $chromS );
  my @maps = ( $linkmap, @rhmaps ); #linkage map first to preserve order;

  my %dataSets = ( dir => "$cd/$iMapDir" );
  $dataSets{maps} = join( "|", @maps );
  $dataSets{table} = $mapTable;
  foreach my $ch ( @chroms ){
    undef my @datas;
    foreach my $m ( @maps ){
      my $data = "$cd/$m/$ch/vectors.rh";
      if( $m =~ /linkage/ ){
	$data = "$cd/$m/$ch/vector.lm";
      }
      push @datas, $data;
      if( -d "$iMapDir/$ch" ){
	my @bfiles = glob( "$iMapDir/$ch/*" );
        my @gbfiles = glob( "$iMapDir/$ch/grid/*/*" );
        my @gfiles = glob( "$iMapDir/$ch/grid/*" );
        my @tfiles = glob( "$iMapDir/$ch/cgtmp*/*" );
        my @files = ( @tfiles, @gbfiles, @gfiles, @bfiles );
	undef my @clean;
	foreach my $f ( @files ){
	  push @clean, $f;
	  if( $f =~ /pass/ ){
	    pop @clean;
	  }
	  if( $f =~ /draft/ ){
	    pop @clean;
	  }
	}
	if( defined $purge ){
	  @clean = @files;
	}
	foreach my $c ( @clean ){
          if( -f $c ){ unlink $c; }
          if( -d $c ){ rmdir $c; }
	}
      } else {
	mkdir "$iMapDir/$ch";
      }
    }
    if( $RUN_ON_GRID ){
      if( ! -d "$iMapDir/$ch/grid" ){
	mkdir "$iMapDir/$ch/grid";
	mkdir "$iMapDir/$ch/grid/scripts";
	print STDERR "cp $main::CARTA_EM_BIN $iMapDir/$ch/grid\n";
	system ( "cp $main::CARTA_EM_BIN $iMapDir/$ch/grid" );
      }
    }
    $dataSets{$ch} = [ @datas ];
  }
  return( %dataSets );
}

sub loadDataByOrder {
  my $df = shift;
  my @files = @{$df};
  my $loaded = 0;

  undef my %mapSets;
  foreach my $file ( @files ){
    my @fs = split( /\//, $file );
    my $map = $fs[$#fs-2];
    dsLoad( $file );
    $loaded++;
    $mapSets{$map} = $loaded;
  }

  dsMergOr( 1, 2 );
  my $merged = $loaded;
  for( my $i=1; $i+1 < $loaded; $i++ ){
    dsMergOr( $i+2, $i+$loaded );
  }
  return $loaded;
}

sub loadData {
  my $df = shift;
  my @files = @{$df};
  my $loaded = 0;

  undef my %mapSets;
  foreach my $file ( @files ){
    my @fs = split( /\//, $file );
    my $map = $fs[$#fs-2];
    dsLoad( $file );
    $loaded++;
    $mapSets{$map} = $loaded;
  }

  dsMerge( 1, 2 );
  my $merged = $loaded;
  for( my $i=1; $i+1 < $loaded; $i++ ){
    dsMerge( $i+2, $i+$loaded );
  }
  return $loaded;
}

sub buildMap {
  my $df = shift;
  my $rhext = shift;
  my $mapList = shift;
  my $mapTable = shift;
  chomp ( my $dir = `pwd` );

  my @loc = split( /\//, $dir );
  my $chr = $loc[$#loc];

  my @dataFiles = @{$df};
  for my $i ( 0 .. $#dataFiles ){
    if( $dataFiles[$i] =~ /vectors.rh/ ){
      $dataFiles[$i] =~ s/vectors.rh/vectors.$rhext/ ;
    }
  }

  my @singleSetOrder = ( 1 .. scalar @dataFiles ); 
  CGrestart();

  undef my $loaded;
  if( $mergeBy eq 'order' ){
    $loaded = loadDataByOrder( \@dataFiles );
  } else {
    $loaded = loadData ( \@dataFiles );
  }

  redirTcl( "markerInfo.merged" );
  mrkInfo();
  redirStd();

  my %markerSets = getMarkers( $loaded, "markerInfo.merged" );
  my %iId = markerId( "markerInfo.merged" );

  undef my %commonMarkers;
  my %allMarkers = %iId;
  foreach my $m ( keys %iId ){
    if( scalar @{$markerSets{$m}} ==  $loaded ){
      $commonMarkers{$m} = $iId{$m};
    }
  }

  my @nameOrder = sort { $commonMarkers{$a} <=> $commonMarkers{$b} }
                        keys %commonMarkers;
  undef my @numOrder;
  foreach my $m ( @nameOrder ){
    push @numOrder, $commonMarkers{$m};
  }

  setMrkOrd( join( ' ', @numOrder ) );
  sEM();
  printMap(0);

  if( -s "draft" ){
    print "have draft\n";
    CGrestart();
    undef my $loaded;
    if( $mergeBy eq 'order' ){
      $loaded = loadDataByOrder( \@dataFiles );
    } else {
      $loaded = loadData ( \@dataFiles );
    }
    my %curOrd = readMap( "draft" );
    undef my @order;
    foreach my $m ( sort { $curOrd{$a} <=> $curOrd{$b} } keys %curOrd ) {
      if( defined $iId{$m} ){
	push @order, $iId{$m};
      } else {
	die "id not defined $m\n";
      }
    }
    setMrkOrd( join( ' ', @order ) );
    my $curLike = sEM();
    saveMap( {map => 0, file => "map.$rhext" } );
    if( defined $mapTable ){
      if( $curOrd{$nameOrder[0]} < $curOrd{$nameOrder[$#nameOrder]} ){
	print "same orientation\n";
	storeMap( $rhext, "map.$rhext", $chr, $mapList, $mapTable );
      } else {
	print "flip orientation\n";
	storeMap( $rhext, "map.$rhext.rev", $chr, $mapList, $mapTable );
      }
    }
    return;
  }
print "directory $dir\n";

  # framework map from markers common to all data
  undef my %mapped;
  undef my %tried;
  my %unmapped = %iId;
  my $addPass = 0;

  if ( ! -s "fwMap" ) {
    my @fwPair = ( $numOrder[0], $numOrder[1] ); #start with 1st two common 
                                                 # markers on linkage map
    buildfw( 0, 0, join( " ", @fwPair ) , 0 );
    my $fwLike = sEM();
    undef my %curBest;
    $curBest{0} = $fwLike;
    saveMap( {map => 0, file => "fwMap" } );
    if( defined $mapTable ){
      storeMap( "fwMap", "fwMap", $chr, $mapList, $mapTable );
    }
    printMap(0);
    my %frameOrd = readMap( "fwMap" );
    foreach my $f ( keys %frameOrd ) {
      if ( defined $iId{$f} ) {
	delete $unmapped{$f};
	$mapped{$f} = $frameOrd{$f};
      }
    }
  }

  my @oldMaps = glob( "pass.*.pf" );

  if( scalar @oldMaps >= 1 ){  # at least one map built earlier
    my $maxI = -1;
    foreach my $m ( @oldMaps ){
      my @md = split( /\./, $m );
      $maxI = $md[1] > $maxI ? $md[1] : $maxI;
    }
    if( $maxI >= 0 ){
      $addPass = $maxI+1;
      my %curOrd = readMap( "pass.$maxI.pf" );
      foreach my $f ( keys %curOrd ) {
	if ( defined $iId{$f} ) {
	  delete $unmapped{$f};
	  $mapped{$f} = $curOrd{$f};
	  print "$f $iId{$f} $mapped{$f}\n";
	}
      }
    }
  }

  print "mapped ", scalar keys %mapped, "\n";
  %tried = %mapped;

  # add remaining markers

  while( scalar keys %unmapped > scalar keys %tried ){

    print "starting loop unmapped ", scalar keys %unmapped;
    print " tried ", scalar keys %tried, "\n";
    undef my %add;
    my $commonSets = 0;
    my $markRead = 0;
    my $singleSet = 0;
    my $triedFW = 0;

    if( scalar keys %tried > 0 ){ #retry previously tried after adding other
      undef my @mNumOrd;
      undef my @mNamOrd;
      foreach my $m ( sort { $mapped{$a} <=> $mapped{$b} } keys %mapped ) {
	push @mNamOrd, $m;
	push @mNumOrd, $iId{$m};
        if( exists $tried{$m} ){
          delete $tried{$m};
        }
      }

      undef my @aNumOrd;
      undef my @aNamOrd;
      foreach my $m ( sort { $tried{$b} <=> $tried{$a} } keys %tried ){
          push @aNamOrd, $m;
          push @aNumOrd, $iId{$m};
      }
      my @order = ( @mNumOrd, @aNumOrd );
      if( scalar @order > scalar @mNumOrd ){
        setMrkOrd( join( ' ', @order ) );
        buildfw( 0, .01, join( " ", @mNumOrd ) , 0 );
        saveMap( {map => 0, file => "retry.$addPass" } );
        printMap(0);
        my %curOrd = readMap( "retry.$addPass" );
        if( scalar keys %curOrd > scalar keys %mapped ){
          undef @order;
          foreach my $m ( sort { $curOrd{$a} <=> $curOrd{$b} } keys %curOrd ) {
            if( defined $iId{$m} ){
              push @order, $iId{$m};
            } else {
              print "id not defined $m\n";
            }
          }
          setMrkOrd( join( ' ', @order ) );
          my $curLike = sEM();
          my $pfMap = polishAndFlip( $curLike, 6 );

          %curOrd = readMap( $pfMap );
          foreach my $f ( keys %curOrd ) {
            if ( defined $iId{$f} ) {
              delete $unmapped{$f};
              delete $tried{$f};
              $mapped{$f} = $curOrd{$f};
            }
          }
          my %best = getCurrentBest();
          my ( $bestNum ) = keys %best;
          saveMap( { map => $bestNum, file => "retry.$addPass.pf" } );
        }
      }
    }

    foreach my $m ( sort { scalar @{$markerSets{$b}} * 10000 - $iId{$b} <=>
                           scalar @{$markerSets{$a}} * 10000 - $iId{$a} }
                    keys %iId ){

      my @a = @{$markerSets{$m}};
print "$m @a\n";

      if ( !(defined $mapped{$m}) && !defined( $tried{$m} ) ) {
        if( $markRead == 0 ){ 
          $commonSets = scalar @a; 
          if( $commonSets == 1 ){
            #$singleSet = $a[0];
            $singleSet = pop @singleSetOrder;
          }
        }
        $markRead++;
        if( scalar @a > 1 ){
          if( scalar @a == $commonSets ){
            $add{$m} = $iId{$m};
            $tried{$m} = $iId{$m};
          }
        } else {
          if( $a[0] == $singleSet ){
            $add{$m} = $iId{$m};
            $tried{$m} = $iId{$m};
          }
        }
      }
    }
    if ( scalar keys %add > 0 ) {
      undef my @mNumOrd;
      undef my @mNamOrd;
      foreach my $m ( sort { $mapped{$a} <=> $mapped{$b} } keys %mapped ) {
	push @mNamOrd, $m;
	push @mNumOrd, $iId{$m};
      }

      undef my @aNumOrd;
      undef my @aNamOrd;
      foreach my $m ( sort { scalar @{$markerSets{$b}} * 10000 - $add{$b} <=>
                             scalar @{$markerSets{$a}} * 10000 - $add{$a} }
                      keys %add ) {
	push @aNamOrd, $m;
	push @aNumOrd, $iId{$m};
      }

      my @allAddNumOrd = @aNumOrd;
      my $numAvail = scalar @allAddNumOrd;
      my $numToAdd = $numAvail;
      my $denom = 2;
      while( $numToAdd > $checkpointFrequency ){
        $numToAdd /= $denom;
        $denom++;
      }
      $denom--;
      undef @aNumOrd;
      for ( my $i=0; $i<$numAvail; $i++ ){
        if( $i % $denom == 0 ){ # separated, not consecutive markers
                                # if data sets ordered by map order
          push @aNumOrd, $allAddNumOrd[$i];
        }
      }

      print "avail ", $numAvail;
      print " add ",scalar @aNumOrd, " to ", scalar @mNumOrd;
      print " all ", scalar keys %iId, "\n";
      print "commonSets $commonSets tryFW $triedFW\n"; 


      my @order = ( @mNumOrd, @aNumOrd );
      setMrkOrd( join( ' ', @order ) );
      buildfw( 0, .01, join( " ", @mNumOrd ) , 0 ); # keep markers with unambiguous
                                                    # position on map
						    # .01 can be changed to
						    # a more stringent criteria
						    # if desired
      saveMap( {map => 0, file => "pass.$addPass" } );
      printMap(0);
      my %curOrd = readMap( "pass.$addPass" );
      undef @order;
      foreach my $m ( sort { $curOrd{$a} <=> $curOrd{$b} } keys %curOrd ) {
        if( defined $iId{$m} ){
          push @order, $iId{$m};
        } else {
          print "id not defined $m\n";
        }
      }
      setMrkOrd( join( ' ', @order ) );
      my $curLike = sEM();
      my $pfMap = polishAndFlip( $curLike, 6 );

      %curOrd = readMap( $pfMap );
      foreach my $f ( keys %curOrd ) {
	if ( defined $iId{$f} ) {
	  delete $unmapped{$f};
          delete $tried{$f};
	  $mapped{$f} = $curOrd{$f};
	}
      }
      my %best = getCurrentBest();
      my ( $bestNum ) = keys %best;
      saveMap( { map => $bestNum, file => "pass.$addPass.pf" } );

      $addPass++;
    }
  }

  if ( scalar keys %tried > 0 ) { # final retry for unambiguous fw
    undef my @mNumOrd;
    undef my @mNamOrd;
    foreach my $m ( sort { $mapped{$a} <=> $mapped{$b} } keys %mapped ) {
      push @mNamOrd, $m;
      push @mNumOrd, $iId{$m};
      if ( exists $tried{$m} ) {
        delete $tried{$m};
      }
    }

    undef my @aNumOrd;
    undef my @aNamOrd;
    foreach my $m ( sort { $tried{$a} <=> $tried{$b} } keys %tried ) {
      push @aNamOrd, $m;
      push @aNumOrd, $iId{$m};
    }
    my @order = ( @mNumOrd, @aNumOrd );
    if ( scalar @order > scalar @mNumOrd ) {
      setMrkOrd( join( ' ', @order ) );
      buildfw( 0, .01, join( " ", @mNumOrd ) , 0 );  # retry unambiguous placement
                                                     # framework criteria (.01) 
						     # sould match criteria set 
						     # above (This really should
						     # be done with a variable...)
      saveMap( {map => 0, file => "retry.$addPass" } );
      printMap(0);
      my %curOrd = readMap( "retry.$addPass" );
      if ( scalar keys %curOrd > scalar keys %mapped ) {
        undef @order;
        foreach my $m ( sort { $curOrd{$a} <=> $curOrd{$b} } keys %curOrd ) {
          if ( defined $iId{$m} ) {
            push @order, $iId{$m};
          } else {
            print "id not defined $m\n";
          }
        }
        setMrkOrd( join( ' ', @order ) );
        my $curLike = sEM();
        my $pfMap = polishAndFlip( $curLike, 6 );

        %curOrd = readMap( $pfMap );
        foreach my $f ( keys %curOrd ) {
          if ( defined $iId{$f} ) {
            delete $unmapped{$f};
            delete $tried{$f};
            $mapped{$f} = $curOrd{$f};
          }
        }
        my %best = getCurrentBest();
        my ( $bestNum ) = keys %best;
        saveMap( { map => $bestNum, file => "retry.$addPass.pf" } );
      }
    }
  }

  my %best = getCurrentBest();
  my ( $bestNum ) = keys %best;
  saveMap( { map => $bestNum, file => "framework.final" } );
  if( defined $mapTable ){
    storeMap( "dFW", "framework.final", $chr, $mapList, $mapTable );
  }

  #my $lastPass = $addPass-1;
  #storeMap( "dFW", "pass.$lastPass.pf", $chr );
  my %curOrd = readMap( "framework.final" );
  print "framework built ", scalar keys %curOrd,
        " still have " , scalar keys %unmapped, " not mapped\n";


  undef my @order;
  undef my @fwOrder;
  foreach my $m ( sort { $curOrd{$a} <=> $curOrd{$b} } keys %curOrd ) {
    if( defined $iId{$m} ){
      push @fwOrder, $iId{$m};
    } else {
      print "id not defined $m\n";
    }
  }

  undef my @lastTry;
  foreach my $m ( sort { $unmapped{$a} <=> $unmapped{$b} } keys %unmapped ){
      push @lastTry, $iId{$m};
  }

  @order = ( @fwOrder, @lastTry );
  setMrkOrd( join( ' ', @order  ) );
  buildfw( 0, 0, join( " ", @fwOrder ) , 0 ); #add everything else...
  saveMap( {map => 0, file => "all.add" } );
  %curOrd = readMap( "all.add" );
  undef @order;
  foreach my $m ( sort { $curOrd{$a} <=> $curOrd{$b} } keys %curOrd ) {
    if( defined $iId{$m} ){
      push @order, $iId{$m};
    } else {
      print "id not defined $m\n";
    }
  }
  setMrkOrd( join( ' ', @order ) );
  my $curLike = sEM();
  my $pfMap = polishAndFlip( $curLike, 6 );

  %curOrd = readMap( $pfMap );
  foreach my $f ( keys %curOrd ) {
    if ( defined $iId{$f} ) {
      delete $unmapped{$f};
      delete $tried{$f};
      $mapped{$f} = $curOrd{$f};
    }
  }
  %best = getCurrentBest();
  ( $bestNum ) = keys %best;
  saveMap( { map => $bestNum, file => "all.pf" } );
  saveMap( { map => $bestNum, file => "draft" } );
  if( defined $mapTable ){
    storeMap( "draft", "draft", $chr, $mapList, $mapTable );
  }

  print "mapped ", scalar keys %mapped;
  print "unmapped ", scalar keys %unmapped;
  print "tried ", scalar keys %tried;
  print "\n";

}

sub storeMap {
  my $map_id = shift;
  my $mapFile = shift;
  my $chr = shift;
  my $mapList = shift;
  my $mapTable = shift;

  $chr =~ s/X/30/;
  $chr =~ s/Y/31/;

  my $con = connectPg();

  my $qMap = $con->quote( $map_id );
  $con->do( qq { delete from $mapTable
		 where map_id = $qMap and linkage_group = $chr } );

  my $ih = $con->prepare( qq { insert into $mapTable
			       ( imap_id, map_id, map_scale, 
                                 linkage_group, position, order_in_map )
			       values ( ?, ?, ?, ?, ?, ? ) } );
  my $mid = $con->prepare( "
select marker, map from imap_id where imap_id = ? " );

  my %mapDataSets = dataSets( $mapList, $con );

print "map list $mapList\n";
  foreach my $m ( keys %mapDataSets ){
    print "$m $mapDataSets{$m}{map} $mapDataSets{$m}{shortName}\n";
  }
  
  undef my %order;
  undef my %pos;
  my $maxPos = 0;
  my $cumPos = 0;
  undef my $lastMarker;
  undef my $curDataSet;

  undef my $dataSet;

  undef my @dataSetsMapped;
  open( MF, "<", $mapFile );
  while( <MF> ){
    my $l = $_;
    $l =~ s/^\s+//;
    $l =~ s/\n$//;
    $l =~ s/\s+$//;
    $l =~ s/\s+/ /g;

    if( $l =~ /^Data Set Number/ ){
      my @rec = split( / /, $l );
      $dataSet = $rec[3];
      $curDataSet = $dataSet;
      $maxPos = 0;
      $cumPos = 0;
print "data set now $curDataSet\n";
      push @dataSetsMapped, $dataSet;
    }

    if( $l =~ /^\d/ ){
      my @rec = split( / /, $l );
      if( $l =~ /cR/ || $l =~ /cM/  || $rec[$#rec] =~ /----/ ){
        if( scalar @rec > 2 ) {
          if( scalar @rec > 5 ){
            $order{$rec[2]} = $rec[0];
            $pos{$rec[2]}{$curDataSet} = $cumPos;
            $cumPos += $rec[3]
          }
          if( $rec[$#rec] =~ /----/ ){
            $order{$rec[2]} = $rec[0];
            $pos{$rec[2]}{$curDataSet} = $cumPos;
            $maxPos = $cumPos;
          }
        }
      }
    }
  }
  close( MF );

  foreach my $m ( keys %order ){
    my %p = %{$pos{$m}};
#print $m, " ", $pos{$m}, " ", $order{$m}, "\n";
    foreach my $ds ( keys %p ){
      undef my $scale;
      if( scalar @dataSetsMapped == 1 ){
	$scale = "relative";
      } else {
	$scale = $mapDataSets{$ds}{map};
      }
      my $position = $pos{$m}{$ds};
      #$mid->execute( $m );
      #while( my ( $marker, $dmap ) = $mid->fetchrow_array ){
	$ih->execute( $m, $map_id, $scale, $chr, $position, $order{$m} );
      #}
      #if( exists $storeQ{$ds} ){
      #  #$storeQ{$ds}->execute( $position, "bos", $map, $chr, $m );
      #} else {
      #  #$updgen->execute( $position, "bos", $map, $chr, $m );
      #}
    }
  }
}

  
sub getMarkers {
  my $orig = shift;
  my $mf = shift;

  my $all = $orig + $orig - 1;

  open( MF, "<", $mf );

  undef my %allMarkers;
  undef my %commonMarkers;
  undef my %markerSets;
  my $read = 0;
  while( <MF> ){
    my $l = $_;
    $l =~ s/^\s+//;
    $l =~ s/\n$//;
    $l =~ s/\s+$//;
    $l =~ s/\s+/ /g;
    if( $l =~ /^\d/ ){

      my @sr = split( /\:/, $l );
      $sr[1] =~ s/^\s+//;
      my @ds = split( /\s+/, $sr[1] );
      my @ma = split( / /, $sr[0] );

      $allMarkers{$ma[1]} = $ma[0];

      if( scalar @ds == $all ){
	$commonMarkers{$ma[1]} = $ma[0];
print "common $ma[1] $ma[0]\n";
      }

      undef my @inSet;
      for( my $i=0; $i < $orig; $i++ ){
	if( defined $ds[$i] && $ds[$i] <= $orig ){
	  push @inSet, $ds[$i];
	}
      }
      $markerSets{$ma[1]} = [ @inSet ];
    }
  }
  close( MF );
  return( %markerSets );
}

sub dataSets {
  my $list = shift;
  my $con = shift;

  my @maps = split( /\|/, $list );

  my $nq = $con->prepare( "
select map from imap_maps where short_name = ? or map = ? " );

  undef my %ds;
  my $dsn = 1;

  foreach my $m ( @maps ){
    $nq->execute( $m, $m );
    my( $fn ) = $nq->fetchrow_array;
    $ds{$dsn}{map} = $fn;
    $ds{$dsn}{shortName} = $m;
    $dsn++;
  }
  return( %ds );
}