+ found bug which caused ~10 reads of overlap "0"
[qpalma.git] / tools / data_tools / filterReads.c
1 ////////////////////////////////////////////////////////////
2 // The purpose of this program is to read a gff and a
3 // solexa reads file and create a data set used by QPalma.
4 //
5 //
6 //
7 ////////////////////////////////////////////////////////////
8
9 #include <sys/mman.h>
10 #include <sys/stat.h>
11 #include <stdio.h>
12 #include <stdlib.h>
13 #include <string.h>
14 #include <unistd.h>
15 #include <math.h>
16
17 #include "common.h"
18 #include "datastructures.h"
19
20 int parse_gff(char* filename,FILE* fid,struct gene*** allGenes);
21
22 void process_reads(FILE* reads_fs,struct gene*** allGenes,int numGenes,FILE* out_fs);
23
24 void combine_info(int exon_stop, int exon_start, void** upstream, int up_size, void** downstream, int down_size, FILE* out_fs);
25
26 int fitting(char* up_prb, char* up_prb_end, char* down_prb, char* down_prb_end);
27
28 void remove_ambiguities(char * old_seq, int old_seq_size, char* new_seq);
29
30 static char *info = "Usage is:\n./filterReads gff reads output";
31
32 const int read_size = 36;
33
34 int main(int argc, char* argv[]) {
35
36 if(argc != 4) {
37 printf("%s\n",info);
38 exit(EXIT_FAILURE);
39 }
40
41 int status;
42 int filenameSize = 256;
43 char *gff_filename = malloc(sizeof(char)*filenameSize);
44 char *reads_filename = malloc(sizeof(char)*filenameSize);
45 char *output_filename = malloc(sizeof(char)*filenameSize);
46
47 strncpy(gff_filename,argv[1],filenameSize);
48 strncpy(reads_filename,argv[2],filenameSize);
49 strncpy(output_filename,argv[3],filenameSize);
50
51 FILE *gff_fs = fopen(gff_filename,"r");
52 FILE *reads_fs = fopen(reads_filename,"r");
53 FILE *out_fs = fopen(output_filename,"w");
54
55 if(gff_fs == NULL) {
56 printf("Error: Could not open file: %s",gff_filename);
57 exit(EXIT_FAILURE);
58 }
59
60 if(reads_fs == NULL) {
61 printf("Error: Could not open file: %s",reads_filename);
62 exit(EXIT_FAILURE);
63 }
64
65 if(out_fs == NULL) {
66 printf("Error: Could not open file: %s",output_filename);
67 exit(EXIT_FAILURE);
68 }
69
70 struct gene** allGenes;
71 int numGenes = parse_gff(gff_filename,gff_fs,&allGenes);
72 status = fclose(gff_fs);
73 if(status != 0)
74 printf("closing of gff filestream failed!\n");
75
76 printf("Successfully parsed gff file! Found %d genes.\n",numGenes);
77
78 process_reads(reads_fs,&allGenes,numGenes,out_fs);
79
80 status = fclose(reads_fs);
81 status = fclose(out_fs);
82 if(status != 0)
83 perror("fclose");
84
85 //free(allGenes);
86 free(gff_filename);
87 free(reads_filename);
88 return 0;
89 }
90
91 void process_reads(FILE* reads_fs,struct gene*** allGenes,int numGenes, FILE* out_fs) {
92 int status;
93
94 int buffer_size= 64;
95 int chr = 0;
96 int pos = 0;
97 char* seq = malloc(sizeof(char)*buffer_size);
98 unsigned long id = 0;
99 char strand = ' ';
100 int mismatch = 0;
101 int occurrence = 0;
102 int size = 0;
103 int cut = 0;
104 char* prb = malloc(sizeof(char)*buffer_size);
105 char* cal_prb = malloc(sizeof(char)*buffer_size);
106 char* chastity = malloc(sizeof(char)*buffer_size);
107
108 int reads_fid = fileno(reads_fs);
109 struct stat reads_stat;
110 if ( fstat(reads_fid,&reads_stat) == -1) {
111 perror("fstat");
112 exit(EXIT_FAILURE);
113 }
114 off_t reads_filesize = reads_stat.st_size;
115 printf("Reads file is of size %lu bytes\n",(unsigned long) reads_filesize);
116 int numReads = reads_filesize / 178.0;
117
118 void *reads_area = mmap (NULL,reads_filesize,PROT_READ|PROT_WRITE,MAP_PRIVATE,reads_fid,0);
119 if (reads_area == MAP_FAILED) {
120 perror("mmap");
121 exit(EXIT_FAILURE);
122 }
123 close(reads_fid);
124 printf("Successfully mapped %lu bytes of reads file into memory\n",(unsigned long)reads_filesize);
125
126 void* linePtr = reads_area;
127 char* current_line = malloc(sizeof(char)*256);
128
129 int SIZE = 500;
130 // initialize boundary arrays
131 void** upstream_end = malloc(sizeof(void*)*SIZE);
132 void** upstream_overlap = malloc(sizeof(void*)*SIZE);
133 void** downstream_start = malloc(sizeof(void*)*SIZE);
134 void** downstream_overlap= malloc(sizeof(void*)*SIZE);
135 int ue,uo,ds,dov;
136 ue = uo = ds = dov = 0;
137
138 int skippedLinesCounter = 0;
139
140 int prev_exon_stop = -1;
141 int cur_exon_start = -1;
142
143 current_line = strncpy(current_line,linePtr,256);
144 int gene_idx = 0;
145 int exon_idx = 1;
146 struct gene* currentGene = (*allGenes)[gene_idx];
147
148 int readCtr = 0;
149 int old_gene_stop = -1;
150 int old_pos = 0;
151 char* posPtr;
152 int new_chr = 0;
153 int new_pos = 0;
154 char* tmp_line = malloc(sizeof(char)*256);
155 // start of the parsing loop
156 while(1) {
157 if (gene_idx == numGenes || strcmp(current_line,"") == 0)
158 break;
159
160 if (readCtr != 0 && readCtr % 1000000 == 0)
161 printf("Processed %d/%d genes and %d/%d reads.\n",readCtr,gene_idx,numGenes,readCtr,numReads);
162
163 //if (gene_idx >= 1833)
164 // printf("currentGene start/stop: %d/%d. Positions is %d\n",currentGene->start,currentGene->stop,pos);
165
166 old_pos = pos;
167 status = sscanf(current_line,"%d\t%d\t%s\t%lu\t%c\t%d\t%d\t%d\t%d\t%s\t%s\t%s\n",
168 &chr,&pos,seq,&id,&strand,&mismatch,&occurrence,&size,&cut,prb,cal_prb,chastity);
169 if (status < 12) {
170 skippedLinesCounter++;
171 }
172 //printf("gene start/stop %d,%d, pos: %d\n",currentGene->start,currentGene->stop,pos);
173 // if the read is occurring several times elsewhere then get rid of it
174 if(occurrence != 1) {
175 while (*(char*)linePtr != '\n') linePtr++;
176 linePtr++;
177 readCtr += 1;
178 current_line = strncpy(current_line,linePtr,256);
179 continue;
180 }
181
182 if (!(currentGene->start <= pos && (pos + read_size-1) <= currentGene->stop)) { // read is not within gene borders
183
184 if ( currentGene->stop < (pos + read_size-1) || currentGene->start < old_gene_stop ) { // go to next gene
185 gene_idx++;
186 exon_idx = 1;
187 old_gene_stop = currentGene->stop ;
188 currentGene = (*allGenes)[gene_idx];
189 //printf("currentGene->start / currentGene->stop %d/%d pos is %d\n",currentGene->start,currentGene->stop,pos);
190 ue = uo = ds = dov = 0;
191 continue;
192 }
193
194 if ( pos < currentGene->start || pos < old_pos) { // go to next read
195 next_read:
196
197 //posPtr = linePtr;
198 //while (1) {
199 // printf("posPtr points to %c\n",*(char*)posPtr);
200 // if ((*(char*)posPtr) == '\n') {
201 // posPtr++;
202 // tmp_line = strncpy(tmp_line,posPtr,256);
203
204 // if (strcmp(tmp_line,"") == 0)
205 // break;
206
207 // sscanf(tmp_line,"%d\t%d\t",new_chr,new_pos);
208 // printf("new_pos %d\n",new_pos);
209 // if (new_pos >= currentGene->start) {
210 // linePtr = posPtr;
211 // break;
212 // }
213 // }
214 // posPtr++;
215 //}
216 //printf("Went out!\n");
217
218 while (*(char*)linePtr != '\n') linePtr++;
219 linePtr++;
220 readCtr += 1;
221 current_line = strncpy(current_line,linePtr,256);
222
223 continue;
224 }
225
226 } else { // read IS within gene borders
227
228 exon_label:
229
230 if (exon_idx == currentGene->num_exons) {
231 gene_idx++;
232 exon_idx = 1;
233 currentGene = (*allGenes)[gene_idx];
234 continue;
235 }
236
237 prev_exon_stop = currentGene->exon_stops[exon_idx-1];
238 cur_exon_start = currentGene->exon_starts[exon_idx];
239
240 if (cur_exon_start - prev_exon_stop < 6 || cur_exon_start < pos ) { // go to next exon
241 exon_idx++;
242
243 if (ue != 0 && dov != 0)
244 combine_info(prev_exon_stop,cur_exon_start,upstream_end,ue,downstream_overlap,dov,out_fs);
245
246 if (uo != 0 && ds != 0)
247 combine_info(prev_exon_stop,cur_exon_start,upstream_overlap,uo,downstream_start,ds,out_fs);
248
249 ue = uo = ds = dov = 0;
250 goto exon_label;
251 }
252
253 if ( pos + (read_size-1) < prev_exon_stop ) // go to next read
254 goto next_read;
255
256 // if this position is reached the read is somehow overlapping or
257 // exactly on the exon boundary. now determine where exactly:
258 if (pos + (read_size-1) == prev_exon_stop) { // read ends at previous exon end
259 upstream_end[ue] = linePtr;
260 ue++;
261 goto next_read;
262 }
263
264 if ( (prev_exon_stop - pos) >= 6 && (prev_exon_stop - pos) <= 30) { // read overlaps with previous exon boundary
265 upstream_overlap[uo] = linePtr;
266 uo++;
267 goto next_read;
268 }
269
270 if ( pos == cur_exon_start ) { // read starts at current exon start
271 downstream_start[ds] = linePtr;
272 ds++;
273 goto next_read;
274 }
275
276 if ( (cur_exon_start - pos) >= 6 && (cur_exon_start - pos) <= 30 ) { // read overlaps with current exon boundary
277 downstream_overlap[dov] = linePtr;
278 dov++;
279 goto next_read;
280 }
281
282 goto next_read; // read was not useful i.e. not overlapping/starting at exon boundaries
283 }
284 }
285
286 combine_info(prev_exon_stop,cur_exon_start,upstream_end,ue,downstream_overlap,dov,out_fs);
287 combine_info(prev_exon_stop,cur_exon_start,upstream_overlap,uo,downstream_start,ds,out_fs);
288
289 free(upstream_end);
290 free(upstream_overlap);
291 free(downstream_start);
292 free(downstream_overlap);
293
294 printf("Processed %d reads in total while %d reads where skipped.\n",readCtr,skippedLinesCounter);
295
296 status = munmap(reads_area,reads_filesize);
297 if(status != 0)
298 printf("munmap failed!\n");
299
300 free(seq);
301 free(prb);
302 free(cal_prb);
303 free(chastity);
304 }
305
306 void combine_info(int exon_stop, int exon_start, void** upstream, int up_size, void** downstream, int down_size,FILE* out_fs) {
307 int up_idx, down_idx, status;
308 char* upstream_line = malloc(sizeof(char)*256);
309 char* downstream_line = malloc(sizeof(char)*256);
310
311 int buffer_size= 64;
312
313 int up_chr = 0;
314 int up_pos = 0;
315 char* up_seq = malloc(sizeof(char)*buffer_size);
316 int up_id = 0;
317 char up_strand = ' ';
318 int up_mismatch = 0;
319 int up_occurrence = 0;
320 int up_sz = 0;
321 int up_cut = 0;
322 char* up_prb = malloc(sizeof(char)*buffer_size);
323 char* up_cal_prb = malloc(sizeof(char)*buffer_size);
324 char* up_chastity = malloc(sizeof(char)*buffer_size);
325
326 int down_chr = 0;
327 int down_pos = 0;
328 char* down_seq = malloc(sizeof(char)*buffer_size);
329 int down_id = 0;
330 char down_strand = ' ';
331 int down_mismatch = 0;
332 int down_occurrence = 0;
333 int down_sz = 0;
334 int down_cut = 0;
335 char* down_prb = malloc(sizeof(char)*buffer_size);
336 char* down_cal_prb = malloc(sizeof(char)*buffer_size);
337 char* down_chastity = malloc(sizeof(char)*buffer_size);
338
339 int new_chr = 0;
340 char* new_seq = malloc(sizeof(char)*buffer_size);
341 char new_strand = ' ';
342 char* new_prb = malloc(sizeof(char)*buffer_size);
343 char* new_cal_prb = malloc(sizeof(char)*buffer_size);
344 char* new_chastity = malloc(sizeof(char)*buffer_size);
345
346 up_idx=0;
347 down_idx=0;
348 while(1) {
349 if (up_idx == up_size || down_idx == down_size || up_strand != down_strand)
350 break;
351
352 strncpy(upstream_line,upstream[up_idx],256);
353 status = sscanf(upstream_line,"%d\t%d\t%s\t%d\t%c\t%d\t%d\t%d\t%d\t%s\t%s\t%s\n",
354 &up_chr,&up_pos,up_seq,&up_id,&up_strand,&up_mismatch,&up_occurrence,&up_sz,
355 &up_cut,up_prb,up_cal_prb,up_chastity);
356
357 strncpy(downstream_line,downstream[down_idx],256);
358 status = sscanf(downstream_line,"%d\t%d\t%s\t%d\t%c\t%d\t%d\t%d\t%d\t%s\t%s\t%s\n",
359 &down_chr,&down_pos,down_seq,&down_id,&down_strand,&down_mismatch,&down_occurrence,&down_sz,
360 &down_cut,down_prb,down_cal_prb,down_chastity);
361
362 char* new_up_seq = malloc(sizeof(char)*read_size);
363 char* new_down_seq = malloc(sizeof(char)*read_size);
364
365 remove_ambiguities(up_seq,strlen(up_seq),new_up_seq);
366 remove_ambiguities(down_seq,strlen(down_seq),new_down_seq);
367
368 new_seq[0] = '\0';
369 new_prb[0] = '\0';
370 new_cal_prb[0] = '\0';
371 new_chastity[0] = '\0';
372
373 int fit;
374 int w_size = 6;
375 int overlap = 0;
376 if (up_pos+35 == exon_stop) { // merge with read which is downstream overlapping
377 overlap = exon_start - down_pos;
378
379 fit = fitting(up_prb+(36-w_size),up_prb+36,down_prb+overlap,down_prb+overlap+w_size);
380 if (fit != 1)
381 goto end;
382
383 new_chr = up_chr;
384 new_strand = up_strand;
385
386 strncat(new_seq,new_up_seq+(36-overlap),overlap);
387 strncat(new_prb,up_prb+(36-overlap),overlap);
388 strncat(new_cal_prb,up_cal_prb+(36-overlap),overlap);
389 strncat(new_chastity,up_chastity+(36-overlap),overlap);
390
391 strncat(new_seq,new_down_seq+overlap,36-overlap);
392 strncat(new_prb,down_prb+overlap,36-overlap);
393 strncat(new_cal_prb,down_cal_prb+overlap,36-overlap);
394 strncat(new_chastity,down_chastity+overlap,36-overlap);
395
396 //printf("Between exon stop/start %d/%d : merging pos %d %d with overlap %d\n",exon_stop,exon_start,up_pos+35,down_pos, overlap);
397
398 } // merge with read which is upstream overlapping
399
400 if (down_pos == exon_start) {
401 overlap = up_pos+36 - exon_stop;
402 //printf("overlap is %d\n",overlap);
403 //printf("pos are: %d %d\n",up_pos,down_pos);
404 fit = fitting(up_prb+36-overlap-w_size,up_prb+36-overlap,down_prb,down_prb+w_size);
405 if (fit != 1)
406 goto end;
407
408 new_chr = up_chr;
409 new_strand = up_strand;
410
411 strncat(new_seq,new_up_seq,(36-overlap));
412 strncat(new_prb,up_prb,(36-overlap));
413 strncat(new_cal_prb,up_cal_prb,(36-overlap));
414 strncat(new_chastity,up_chastity,(36-overlap));
415
416 strncat(new_seq,new_down_seq,overlap);
417 strncat(new_prb,down_prb,overlap);
418 strncat(new_cal_prb,down_cal_prb,overlap);
419 strncat(new_chastity,down_chastity,overlap);
420
421 //printf("Between exon stop/start %d/%d : merging pos %d %d with overlap %d\n",exon_stop,exon_start,up_pos,down_pos, overlap);
422 }
423
424 if ( !(up_pos+35 == exon_stop) && !(down_pos == exon_start) )
425 printf("ERROR: Between exon stop/start %d/%d : merging pos %d %d with overlap %d\n",exon_stop,exon_start,up_pos,down_pos, overlap);
426
427 fprintf(out_fs,"%d\t%c\t%s\t%d\t%s\t%s\t%s\n",
428 new_chr,new_strand,new_seq,read_size,new_prb,new_cal_prb,new_chastity);
429
430 end:
431
432 up_idx++;
433 down_idx++;
434
435 free(new_up_seq);
436 free(new_down_seq);
437 }
438 }
439
440 int fitting(char* up_prb, char* up_prb_end, char* down_prb, char* down_prb_end) {
441 double epsilon_mean = 15.0;
442 double epsilon_var = 10.0;
443 double mean_up = 0;
444 double variance_up = 0;
445 double mean_down = 0;
446 double variance_down = 0;
447
448 char *up_ptr = up_prb;
449 char *down_ptr = down_prb;
450
451 int w_size = 0;
452 while(up_ptr != up_prb_end) {
453 mean_up += (*up_ptr)-50;
454 mean_down += (*down_ptr)-50;
455 w_size++;
456 up_ptr++;
457 down_ptr++;
458 }
459 mean_up /= w_size;
460 mean_down /= w_size;
461
462
463 up_ptr = up_prb;
464 down_ptr = down_prb;
465 w_size = 0;
466 while(up_ptr != up_prb_end) {
467 variance_up += pow((*up_prb)-50 - mean_up,2);
468 variance_down += pow((*down_prb)-50 - mean_down,2);
469 w_size++;
470 up_ptr++;
471 down_ptr++;
472 }
473 variance_up /= (w_size-1);
474 variance_down /= (w_size-1);
475
476 //printf("means: %f %f, variances: %f %f\n",mean_up,mean_down,variance_up,variance_down);
477
478 if ( abs(mean_up - mean_down) < epsilon_mean && abs(variance_up - variance_down) < epsilon_var )
479 return 1;
480
481 return 0;
482 }
483
484 void remove_ambiguities(char * old_seq, int old_seq_size, char* new_seq) {
485 //printf("old seq: %s\n",old_seq);
486 //printf("new seq: %s\n",new_seq);
487
488 int idx=0;
489 int new_idx = 0;
490 while(idx<old_seq_size) {
491 if (old_seq[idx] == '[') {
492 new_seq[new_idx] = old_seq[++idx];
493 new_idx++;
494 idx += 3;
495 continue;
496 }
497
498 new_seq[new_idx] = old_seq[idx++];
499 new_idx++;
500 }
501 //printf("old seq: %s\n",old_seq);
502 //printf("new seq: %s\n",new_seq);
503 }
504
505 /*
506 * TODO:
507 * - Check strand -> done simple (only if equal)
508 * - check for [AC] and similar entries -> done simple (see function
509 * remove_ambiguities (exchanges [XY] by the first entry)
510 */