+ fixed alignment index calculation for negative strand
[qpalma.git] / qpalma / sequence_utils.py
index 7cf5e4c..806413f 100644 (file)
@@ -1,6 +1,14 @@
-#!/usr/bin/env python 
-# -*- coding: utf-8 -*- 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Written (W) 2008 Fabio De Bona
+# Copyright (C) 2008 Max-Planck-Society
 
+#
+# This file contains the main interface to the QPalma pipeline.
+#
 #
 # This module holds all functions for queries on the dna flat files and the
 # splice score files.
@@ -17,11 +25,49 @@ import numpy
 
 from numpy.matlib import inf
 
-from Genefinding import *
-from genome_utils import load_genomic
+from Genefinding import doIntervalQuery,new_intp,intp_assign
 
-extended_alphabet = ['-','a','c','g','t','n','[',']']
-alphabet          = ['-','a','c','g','t','n']
+from QPalmaDP import createIntArrayFromList
+
+
+def load_genomic(chromosome, strand, start, stop, genome, one_based=True):
+   """
+   This function stems from Cheng Soon Ongs genome_utils package.
+   """
+
+   if (type(start)==numpy.ndarray) or (type(stop)==numpy.ndarray):
+      assert(len(start) == len(stop))
+      assert((start[1:]-stop[:-1]>0).all())
+      if strand == '+':
+          idx = xrange(len(start))
+      else:
+          idx = xrange(len(start)-1,-1,-1)
+
+      seq = ''.join([load_genomic(chromosome, strand, start[ix], stop[ix], genome)\
+                     for ix in idx])
+      return seq
+  
+      #fname = '/fml/ag-raetsch/share/databases/genomes/' + genome + '/' + chromosome + '.dna.flat'
+      fname = genome + '/' + chromosome + '.dna.flat'
+      f=file(fname)
+      if one_based:
+          f.seek(start-1)
+          str=f.read(stop-start+1)
+      else:
+          f.seek(start)
+          str=f.read(stop-start)
+  
+      if strand=='-':
+          return reverse_complement(str)
+      elif strand=='+':
+          return str
+      else:
+          print 'strand must be + or -'
+          raise KeyError
+
+
+extended_alphabet    = ['-','a','c','g','t','n','[',']']
+alphabet             = ['-','a','c','g','t','n']
 
 
 class DataAccessionException:
@@ -37,7 +83,8 @@ def get_flat_file_size(filename):
       print 'Error occurred while trying to obtain file size'
       raise DataAccessionException
 
-   return int(out)
+   # subtract one because of wc -c output
+   return int(out)-1
 
 
 def reverse_complement(seq):
@@ -112,6 +159,29 @@ def unbracket_seq(seq):
    return "".join(new_seq).lower()
 
 
+def reconstruct_dna_seq(seq):
+   """
+
+   """
+
+   new_seq = ''
+   e = 0
+
+   while True:
+      if e >= len(seq):
+         break
+
+      if seq[e] == '[':
+         new_seq += seq[e+1]
+         e += 4
+      else:
+         new_seq += seq[e]
+         e += 1
+
+   return "".join(new_seq).lower()
+
+
+
 def create_bracket_seq(dna_seq,read_seq):
    """
    This function takes a dna sequence and a read sequence and returns the
@@ -127,64 +197,56 @@ def create_bracket_seq(dna_seq,read_seq):
 
 
 def my_load_genomic(fname, strand, start, stop, one_based=True):
-    if (type(start)==numpy.ndarray) or (type(stop)==numpy.ndarray):
-        assert(len(start) == len(stop))
-        assert((start[1:]-stop[:-1]>0).all())
-        if strand == '+':
-            idx = xrange(len(start))
-        else:
-            idx = xrange(len(start)-1,-1,-1)
-
-        seq = ''.join([load_genomic(chromosome, strand, start[ix], stop[ix], genome)\
-                       for ix in idx])
-        return seq
-
-    f=file(fname)
-    if one_based:
-        f.seek(start-1)
-        str=f.read(stop-start+1)
-    else:
-        f.seek(start)
-        str=f.read(stop-start)
-
-    if strand=='-':
-        return reverse_complement(str)
-    elif strand=='+':
-        return str
-    else:
-        print 'strand must be + or -'
-        raise KeyError
+
+   if (type(start)==numpy.ndarray) or (type(stop)==numpy.ndarray):
+      assert(len(start) == len(stop))
+      assert((start[1:]-stop[:-1]>0).all())
+      if strand == '+':
+         idx = xrange(len(start))
+      else:
+         idx = xrange(len(start)-1,-1,-1)
+
+      seq = ''.join([load_genomic(chromosome, strand, start[ix], stop[ix], genome)\
+      for ix in idx])
+      return seq
+
+   try:
+      f=file(fname)
+      if one_based:
+         f.seek(start-1)
+         str=f.read(stop-start+1)
+      else:
+         f.seek(start)
+         str=f.read(stop-start)
+
+      if strand=='-':
+         return reverse_complement(str)
+      elif strand=='+':
+         return str
+      else:
+         print 'strand must be + or -'
+         raise KeyError
+
+   except:
+      print fname,strand,start,stop
 
 
 class DataAccessWrapper:
    """
    The purpose of this class is to wrap the genomic/splice site score data
    access.
-
-   
-
    """
 
+   def __init__(self,settings):
+      self.genomic_dir        = settings['genome_dir'] 
+      self.acc_score_dir      = settings['acceptor_scores_loc'] 
+      self.don_score_dir      = settings['donor_scores_loc'] 
+      self.genomic_fmt        = settings['genome_file_fmt'] 
+      self.sscore_fmt         = settings['splice_score_file_fmt'] 
 
-   def __init__(self,genomic_data_dir,acc_score_dir,don_score_dir,gen_fmt,sscore_fmt):
-      self.genomic_dir        = genomic_data_dir
-      self.acc_score_dir      = acc_score_dir
-      self.don_score_dir      = don_score_dir
-      self.genomic_fmt        = gen_fmt
-      self.sscore_fmt         = sscore_fmt
 
-      assert os.path.isdir(genomic_data_dir)
-      assert os.path.isdir(acc_score_dir)
-      assert os.path.isdir(don_score_dir)
-
-
-   def get_genomic_fragment_fn(self,id,strand):
-      if strand == '+':
-         return os.path.join(self.genomic_dir,self.genomic_fmt%id)
-      elif strand == '-':
-         return os.path.join(self.genomic_dir,(self.genomic_fmt%id)+'.neg')
-      else:
-         assert False
+   def get_genomic_fragment_fn(self,id):
+      return os.path.join(self.genomic_dir,self.genomic_fmt%id)
 
 
    def get_splice_site_scores_fn(self,id,strand):
@@ -199,21 +261,32 @@ class SeqSpliceInfo():
    
    """
    
-   def __init__(self,accessWrapper,id_list):
+   def __init__(self,accessWrapper,fragment_list):
       self.accessWrapper = accessWrapper
       
       self.chromo_sizes = {}
+
+      # take care that it also works say if we get a chromo_list [1,3,5]
+      # those are mapped then to 0,1,2
+      self.fragment_list = fragment_list
+      self.chromo_map = {}
+      for idx,elem in enumerate(fragment_list):
+         self.chromo_map[elem] = idx
    
       print "Fetching fragment sizes..."
-      for chromo in id_list:
-         filename = self.accessWrapper.get_genomic_fragment_fn(chromo,'+')
+      for id in fragment_list:
+         filename = self.accessWrapper.get_genomic_fragment_fn(id)
          total_size = get_flat_file_size(filename)
-         self.chromo_sizes[chromo] = total_size
+         self.chromo_sizes[self.chromo_map[id]] = total_size
 
       print "done"
 
 
-   def getSpliceScores(self,chromo,strand,intervalBegin,intervalEnd,genomicSeq=''):
+   def getFragmentSize(self,id):
+      return self.chromo_sizes[self.chromo_map[id]]
+
+
+   def getSpliceScores(self,id,strand,intervalBegin,intervalEnd):
       """
       Now we want to use interval_query to get the predicted splice scores trained
       on the TAIR sequence and annotation.
@@ -229,24 +302,21 @@ class SeqSpliceInfo():
       pos_size = new_intp()
       intp_assign(pos_size,1)
 
-      total_size = self.chromo_sizes[chromo]
+      total_size = self.getFragmentSize(id)
 
-      acc_fn, don_fn = self.accessWrapper.get_splice_site_scores_fn(chromo,strand)
+      acc_fn, don_fn = self.accessWrapper.get_splice_site_scores_fn(id,strand)
 
       # fetch acceptor scores
-      acc = doIntervalQuery(chromo,strand,intervalBegin,intervalEnd,acc_fn,total_size)
+      acc = doIntervalQuery(id,strand,intervalBegin,intervalEnd,acc_fn,total_size)
 
       # fetch donor scores
-      don = doIntervalQuery(chromo,strand,intervalBegin,intervalEnd,don_fn,total_size)
+      don = doIntervalQuery(id,strand,intervalBegin,intervalEnd,don_fn,total_size)
 
       return acc, don
 
 
    def fetch_sequence(self,chromo,strand,genomicSeq_start,genomicSeq_stop):
-      #chromo_string  = 'chr%d' % chromo
-      #genomicSeq = load_genomic(chromo_string,strand,genomicSeq_start,genomicSeq_stop,self.flat_files,one_based=False)
-      filename = self.accessWrapper.get_genomic_fragment_fn(chromo,strand)
-      print filename,genomicSeq_start,genomicSeq_stop
+      filename = self.accessWrapper.get_genomic_fragment_fn(chromo)
       genomicSeq = my_load_genomic(filename,'+',genomicSeq_start,genomicSeq_stop,one_based=False)
       genomicSeq = genomicSeq.strip().lower()
 
@@ -260,7 +330,7 @@ class SeqSpliceInfo():
       return genomicSeq
 
 
-   def get_seq_and_scores(self,chromo,strand,genomicSeq_start,genomicSeq_stop,only_seq=False):
+   def get_seq_and_scores(self,id,strand,genomicSeq_start,genomicSeq_stop,only_seq=False,perform_checks=True):
       """
       This function expects an interval, chromosome and strand information and
       returns then the genomic sequence of this interval and the associated scores.
@@ -270,26 +340,20 @@ class SeqSpliceInfo():
       # do not have any score predictions
       NO_SCORE_WINDOW_SIZE = 205
 
+      chromo = self.chromo_map[id]
+
       total_size = self.chromo_sizes[chromo]
 
-      #print genomicSeq_start,genomicSeq_stop
+      print 'genomicSeq: ' + str(genomicSeq_start),str(genomicSeq_stop)
 
-      assert genomicSeq_start < genomicSeq_stop
+      assert genomicSeq_start < genomicSeq_stop, pdb.set_trace()
+      assert genomicSeq_start >= 0
+      assert genomicSeq_stop <= total_size
 
-      if strand == '+':
-         s_start  = genomicSeq_start - 1
-         s_stop   = genomicSeq_stop - 1 
-
-         genomicSeq_start  -= 1
-         genomicSeq_stop   -= 1 
+      genomicSeq = self.fetch_sequence(id,strand,genomicSeq_start,genomicSeq_stop)
 
       if strand == '-':
-         s_start  = total_size - genomicSeq_stop + 1
-         s_stop   = total_size - genomicSeq_start + 1 
-
-      assert genomicSeq_start < genomicSeq_stop
-
-      genomicSeq = self.fetch_sequence(chromo,strand,s_start,s_stop)
+         genomicSeq = reverse_complement(genomicSeq)
 
       if only_seq:
          return genomicSeq
@@ -297,27 +361,29 @@ class SeqSpliceInfo():
       # shorten the intervalEnd to the maximal size of the genomic sequence if it would exceed this size
       lookup_offset_begin  = 10
       lookup_offset_end    = 10
+
       intervalBegin  = max(genomicSeq_start-lookup_offset_begin,0)
       if intervalBegin == 0:
          lookup_offset_begin = 0
+         intervalBegin = genomicSeq_start
          
       intervalEnd    = min(genomicSeq_stop+lookup_offset_end,total_size-1)
       if intervalEnd == total_size-1:
          lookup_offset_end = 0
+         intervalEnd = genomicSeq_stop
 
-      currentAcc, currentDon = self.getSpliceScores(chromo,strand,intervalBegin,intervalEnd,genomicSeq)
-
-      if strand == '-':
-         currentAcc = currentAcc[1:]
-         #currentDon = currentDon[1:]
-         #currentDon = [-inf]+currentDon[:-1]
-      else:
-         currentAcc = currentAcc[2:]
-         currentDon = currentDon[1:]
+      currentAcc, currentDon = self.getSpliceScores(id,strand,intervalBegin,intervalEnd)
 
       # remove the offset positions
-      currentAcc = currentAcc[lookup_offset_begin:]
-      currentDon = currentDon[lookup_offset_begin:]
+      if strand == '+':
+         currentAcc = currentAcc[lookup_offset_begin+2:]
+         currentDon = currentDon[lookup_offset_begin+1:]
+      elif strand == '-':
+         currentAcc = currentAcc[lookup_offset_begin:]
+         currentDon = [-inf]+currentDon
+         currentDon = currentDon[(lookup_offset_begin):]
+      else:
+         assert False
 
       currentAcc = currentAcc[:len(genomicSeq)]
       currentDon = currentDon[:len(genomicSeq)]
@@ -327,12 +393,11 @@ class SeqSpliceInfo():
       currentDon = currentDon+[-inf]*(length-len(currentDon))
       currentDon[-1] = -inf
       
+      assert len(genomicSeq) == len(currentAcc) == len(currentDon), pdb.set_trace()
+
       gt_tuple_pos = [p for p,e in enumerate(genomicSeq) if (p>0 and p<len(genomicSeq)-1 and e=='g' ) and (genomicSeq[p+1]=='t' or genomicSeq[p+1]=='c')]
       ag_tuple_pos = [p for p,e in enumerate(genomicSeq) if p>1 and genomicSeq[p-1]=='a' and genomicSeq[p]=='g']
 
-      #pdb.set_trace()
-      #return genomicSeq, currentAcc, currentDon
-    
       for pos in ag_tuple_pos:
          if pos+intervalBegin < NO_SCORE_WINDOW_SIZE and currentAcc[pos] == -inf:
             currentAcc[pos] = 1e-6
@@ -350,26 +415,147 @@ class SeqSpliceInfo():
       acc_pos = [p for p,e in enumerate(currentAcc) if e != -inf and p > 1]
       don_pos = [p for p,e in enumerate(currentDon) if e != -inf and p > 0]
 
-      print 'ACC: Chromo/strand: %d/%s' % (chromo,strand)
-      print ag_tuple_pos[:10],ag_tuple_pos[-10:]
-      print acc_pos[:10],acc_pos[-10:]
-      if not ag_tuple_pos == acc_pos:
-         print 'ACC: Chromo/strand: %d/%s' % (chromo,strand)
-         print ag_tuple_pos[:10],ag_tuple_pos[-10:]
-         print acc_pos[:10],acc_pos[-10:]
+      check_window_size = 30
+
+      if perform_checks and not ag_tuple_pos == acc_pos:
+         print 'ACC: Chromo/strand: %d/%s' % (id,strand)
+         print ag_tuple_pos[:check_window_size]
+         print acc_pos[:check_window_size]
+         print 
+         print ag_tuple_pos[-check_window_size:]
+         print acc_pos[-check_window_size:]
          pdb.set_trace()
 
-      print 'DON: Chromo/strand: %d/%s' % (chromo,strand)
-      print gt_tuple_pos[:10],gt_tuple_pos[-10:]
-      print don_pos[:10],don_pos[-10:]
-      if not gt_tuple_pos == don_pos:
-         print 'DON: Chromo/strand: %d/%s' % (chromo,strand)
-         print gt_tuple_pos[:10],gt_tuple_pos[-10:]
-         print don_pos[:10],don_pos[-10:]
+      if perform_checks and not gt_tuple_pos == don_pos:
+         print 'DON: Chromo/strand: %d/%s' % (id,strand)
+         print gt_tuple_pos[:check_window_size]
+         print don_pos[:check_window_size]
+         print 
+         print gt_tuple_pos[-check_window_size:]
+         print don_pos[-check_window_size:]
          pdb.set_trace()
 
-      #assert ag_tuple_pos == acc_pos, pdb.set_trace()
-      #assert gt_tuple_pos == don_pos, pdb.set_trace()
-      assert len(genomicSeq) == len(currentAcc) == len(currentDon), pdb.set_trace()
 
       return genomicSeq, currentAcc, currentDon
+
+
+
+
+
+
+##################################
+##################################
+##################################
+##################################
+
+
+   def _get_seq_and_scores(self,id,strand,genomicSeq_start,genomicSeq_stop,only_seq=False,perform_checks=True):
+      """
+      This function expects an interval, chromosome and strand information and
+      returns then the genomic sequence of this interval and the associated scores.
+      """
+
+      chromo = self.chromo_map[id]
+
+      total_size = self.chromo_sizes[chromo]
+
+      if strand == '-':
+         s_start  = total_size - genomicSeq_stop
+         s_stop   = total_size - genomicSeq_start
+         genomicSeq_start = s_start
+         genomicSeq_stop  = s_stop
+
+
+      assert genomicSeq_start < genomicSeq_stop, pdb.set_trace()
+
+      genomicSeq = self.fetch_sequence(id,strand,genomicSeq_start,genomicSeq_stop)
+
+      if strand == '-':
+         genomicSeq = reverse_complement(genomicSeq)
+
+      if only_seq:
+         return genomicSeq
+
+      currentAcc,currentDon = self.get_scores(id,strand,genomicSeq_start,genomicSeq_stop,genomicSeq,total_size,perform_checks)
+
+      return genomicSeq, currentAcc, currentDon
+
+
+   def get_scores(self,id,strand,genomicSeq_start,genomicSeq_stop,genomicSeq,total_size,perform_checks):
+      # Because splice site scores are predicted using a window of the sequence the
+      # first and the last NO_SCORE_WINDOW_SIZE nucleotides of a genomic sequence
+      # do not have any score predictions
+      NO_SCORE_WINDOW_SIZE = 205
+
+      seq_size = len(genomicSeq)
+      # shorten the intervalEnd to the maximal size of the genomic sequence if it would exceed this size
+      lookup_offset_begin  = 10
+      lookup_offset_end    = 10
+
+      intervalBegin  = max(genomicSeq_start-lookup_offset_begin,0)
+      if intervalBegin == 0:
+         lookup_offset_begin = 0
+         
+      intervalEnd    = min(genomicSeq_stop+lookup_offset_end,total_size-1)
+      if intervalEnd == total_size:
+         lookup_offset_end = 0
+
+      print 'interval is: %d %d' % (intervalBegin,intervalEnd)
+
+      # splice score indices are 1-based
+      currentAcc, currentDon = self.getSpliceScores(id,strand,intervalBegin+1,intervalEnd+1)
+
+      print 'original sizes: %d %d' % (len(currentAcc),len(currentDon))
+
+      # remove the offset positions
+      currentAcc = currentAcc[(lookup_offset_begin+1):(lookup_offset_begin+1+seq_size)]
+      currentDon = currentDon[lookup_offset_begin:lookup_offset_begin+seq_size]
+
+      print 'all sizes'
+      print len(genomicSeq)
+      print len(currentAcc)
+      print len(currentDon)
+
+      gt_tuple_pos = [p for p,e in enumerate(genomicSeq) if p<len(genomicSeq)-1 and e=='g' and (genomicSeq[p+1]=='t' or genomicSeq[p+1]=='c')]
+      ag_tuple_pos = [p for p,e in enumerate(genomicSeq) if p>1 and e=='g' and genomicSeq[p-1]=='a' ]
+
+      for pos in ag_tuple_pos:
+         if pos+genomicSeq_start < NO_SCORE_WINDOW_SIZE and currentAcc[pos] == -inf:
+            currentAcc[pos] = 1e-6
+
+         if pos+genomicSeq_start > total_size - NO_SCORE_WINDOW_SIZE and currentAcc[pos] == -inf:
+            currentAcc[pos] = 1e-6
+
+      for pos in gt_tuple_pos:
+         if pos+genomicSeq_start < NO_SCORE_WINDOW_SIZE and currentDon[pos] == -inf:
+            currentDon[pos] = 1e-6
+
+         if pos+genomicSeq_start > total_size - NO_SCORE_WINDOW_SIZE and currentDon[pos] == -inf:
+            currentDon[pos] = 1e-6
+
+      acc_pos = [p for p,e in enumerate(currentAcc) if e != -inf and p > 1]
+      don_pos = [p for p,e in enumerate(currentDon) if e != -inf and p > 0]
+
+      check_window_size = 30
+
+      if perform_checks and not ag_tuple_pos == acc_pos:
+         print 'ACC: Chromo/strand: %d/%s' % (id,strand)
+         print ag_tuple_pos[:check_window_size]
+         print acc_pos[:check_window_size]
+         print 
+         print ag_tuple_pos[-check_window_size:]
+         print acc_pos[-check_window_size:]
+         pdb.set_trace()
+
+      if perform_checks and not gt_tuple_pos == don_pos:
+         print 'DON: Chromo/strand: %d/%s' % (id,strand)
+         print gt_tuple_pos[:check_window_size]
+         print don_pos[:check_window_size]
+         print ''
+         print gt_tuple_pos[-check_window_size:]
+         print don_pos[-check_window_size:]
+         pdb.set_trace()
+
+      assert len(genomicSeq) == len(currentAcc) == len(currentDon), pdb.set_trace()
+
+      return currentAcc,currentDon