1047 lines
45 KiB
C++
1047 lines
45 KiB
C++
#include "serial_md.h"
|
||
|
||
#include <common/hts/bam_buf.h>
|
||
#include <common/utils/debug.h>
|
||
#include <common/utils/global_arg.h>
|
||
#include <common/utils/profiling.h>
|
||
#include <common/utils/timer.h>
|
||
#include <common/utils/util.h>
|
||
#include <sam/utils/read_ends.h>
|
||
|
||
#include <algorithm>
|
||
#include <iostream>
|
||
#include <vector>
|
||
|
||
#include "dup_metrics.h"
|
||
#include "markdups_arg.h"
|
||
#include "md_funcs.h"
|
||
#include "shared_args.h"
|
||
|
||
using std::cout;
|
||
using std::vector;
|
||
|
||
/* 查找 */
|
||
// template<class Itr, class T>
|
||
// static inline Itr binaryFind(Itr first, Itr last, const T &val)
|
||
// {
|
||
// first = std::lower_bound(first, last, val);
|
||
// return (first != last && *first == val) ? first : last;
|
||
// }
|
||
|
||
/* 排序 */
|
||
static inline void sortReadEndsArr(vector<ReadEnds> &arr) {
|
||
size_t blockSize = 64 * 1024;
|
||
if (arr.size() < blockSize) {
|
||
std::sort(arr.begin(), arr.end());
|
||
return;
|
||
}
|
||
size_t blockNum = (arr.size() + blockSize - 1) / blockSize;
|
||
size_t crossNum = 1024;
|
||
size_t start, end, i, left, right;
|
||
std::sort(arr.begin(), arr.begin() + blockSize);
|
||
for (i = 1; i < blockNum; ++i) {
|
||
start = i * blockSize;
|
||
end = min(start + blockSize, arr.size());
|
||
std::sort(arr.begin() + start, arr.begin() + end);
|
||
left = crossNum;
|
||
while (!(arr[start - left] < arr[start])) {
|
||
left = left << 1;
|
||
if (left >= blockSize) {
|
||
std::sort(arr.begin(), arr.end()); // 退化到普通排序
|
||
return;
|
||
}
|
||
}
|
||
right = min(crossNum, end - start - 1);
|
||
|
||
while (!(arr[start - 1] < arr[start + right])) {
|
||
right = min(right << 1, end - start - 1);
|
||
if (right == end - start - 1)
|
||
break;
|
||
}
|
||
std::sort(arr.begin() + start - left, arr.begin() + start + right);
|
||
}
|
||
}
|
||
|
||
/* 处理一组pairend的readends，标记冗余, 这个函数需要串行运行，因为需要做一些统计*/
|
||
static void markDupsForPairs(vector<const ReadEnds *> &vpRe, DPSet<DupInfo> *dupIdx, MDSet<int64_t> *opticalDupIdx,
|
||
DPSet<DupInfo> *repIdx, MDSet<int64_t> *notDupIdx = nullptr,
|
||
MDSet<int64_t> *notOpticalDupIdx = nullptr, MDSet<int64_t> *notRepIdx = nullptr) {
|
||
if (vpRe.size() < 2) {
|
||
if (vpRe.size() == 1) {
|
||
// 这个统计可能会有误差，因为当前位点可能还有没匹配上的read，导致当前位点的read（paired）数量为1
|
||
// 可以通过后续的补充计算来解决这个问题，有必要么？好像没必要
|
||
gMetrics.singletonReads.insert(vpRe[0]->read1IndexInFile);
|
||
}
|
||
return;
|
||
}
|
||
|
||
int maxScore = 0;
|
||
const ReadEnds *pBest = nullptr;
|
||
// bool print = false;
|
||
// int maxOperateTime = 0;
|
||
/** All read ends should have orientation FF, FR, RF, or RR **/
|
||
for (auto pe : vpRe) { // 找分数最高的readend
|
||
// maxOperateTime = max(maxOperateTime, pe->oprateTime);
|
||
// (const_cast<ReadEnds *>(pe))->oprateTime ++;
|
||
if (pe->score > maxScore || pBest == nullptr) {
|
||
maxScore = pe->score;
|
||
pBest = pe;
|
||
}
|
||
/*
|
||
if (pe->read1IndexInFile == 255830545) {
|
||
print = true;
|
||
}
|
||
*/
|
||
}
|
||
/*
|
||
if (print) {
|
||
cout << "mark pair find: " << endl;
|
||
for (auto pe : vpRe) {
|
||
cout << pe->read1IndexInFile << " " << pe->read2IndexInFile << " " << pe->score << endl;
|
||
}
|
||
cout << "mark pair end: " << endl;
|
||
}
|
||
*/
|
||
// cerr << zzhtestnum << " best: " << vpRe.size() << " " << pBest->read1IndexInFile << "-" <<
|
||
// pBest->read2IndexInFile << endl; if (maxOperateTime == 0) ++zzhtestnum;
|
||
if (notDupIdx != nullptr) {
|
||
notDupIdx->insert(pBest->read1IndexInFile);
|
||
notDupIdx->insert(pBest->read2IndexInFile);
|
||
}
|
||
// if (false) {
|
||
// if (true) {
|
||
if (!g_mdArg.READ_NAME_REGEX.empty()) { // 检查光学冗余
|
||
// trackOpticalDuplicates
|
||
vector<const ReadEnds *> prevOpticalRe;
|
||
if (notOpticalDupIdx != nullptr) {
|
||
for (auto pe : vpRe) {
|
||
if (pe->isOpticalDuplicate) {
|
||
prevOpticalRe.push_back(pe);
|
||
}
|
||
}
|
||
}
|
||
trackOpticalDuplicates(vpRe, pBest);
|
||
// 由于重叠问题，可能会更新数据
|
||
if (notOpticalDupIdx != nullptr) {
|
||
for (auto pe : prevOpticalRe) {
|
||
if (!pe->isOpticalDuplicate) {
|
||
notOpticalDupIdx->insert(pe->read1IndexInFile);
|
||
notOpticalDupIdx->insert(pe->read2IndexInFile);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for (auto pe : vpRe) { // 对非best read标记冗余
|
||
if (pe != pBest) { // 非best
|
||
dupIdx->insert(DupInfo(pe->read1IndexInFile, pBest->read1IndexInFile, (int16_t)vpRe.size())); // 添加read1
|
||
if (pe->read2IndexInFile != pe->read1IndexInFile)
|
||
dupIdx->insert(DupInfo(pe->read2IndexInFile, pBest->read1IndexInFile, (int16_t)vpRe.size())); // read2,
|
||
// 注意这里代表是read1的索引
|
||
// 检查是否optical dup
|
||
if (pe->isOpticalDuplicate && opticalDupIdx != nullptr) {
|
||
opticalDupIdx->insert(pe->read1IndexInFile);
|
||
if (pe->read2IndexInFile != pe->read1IndexInFile)
|
||
opticalDupIdx->insert(pe->read2IndexInFile);
|
||
}
|
||
}
|
||
}
|
||
// 在输出的bam文件中添加tag, best作为dupset的代表
|
||
if (g_mdArg.TAG_DUPLICATE_SET_MEMBERS) {
|
||
repIdx->insert(DupInfo(pBest->read1IndexInFile, pBest->read1IndexInFile, (int16_t)vpRe.size()));
|
||
repIdx->insert(DupInfo(pBest->read2IndexInFile, pBest->read1IndexInFile, (int16_t)vpRe.size()));
|
||
if (notRepIdx != nullptr) {
|
||
for (auto pe : vpRe) {
|
||
if (pe != pBest) {
|
||
notRepIdx->insert(pe->read1IndexInFile);
|
||
if (pe->read2IndexInFile != pe->read1IndexInFile)
|
||
notRepIdx->insert(pe->read2IndexInFile);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/* 处理一组非paired的readends，标记冗余 */
|
||
static void markDupsForFrags(vector<const ReadEnds *> &vpRe, bool containsPairs, DPSet<DupInfo> *dupIdx,
|
||
MDSet<int64_t> *notDupIdx = nullptr) {
|
||
if (containsPairs) {
|
||
for (auto pe : vpRe) {
|
||
if (!pe->IsPaired()) {
|
||
dupIdx->insert(pe->read1IndexInFile);
|
||
}
|
||
}
|
||
} else {
|
||
int maxScore = 0;
|
||
const ReadEnds *pBest = nullptr;
|
||
for (auto pe : vpRe) {
|
||
if (pe->score > maxScore || pBest == nullptr) {
|
||
maxScore = pe->score;
|
||
pBest = pe;
|
||
}
|
||
}
|
||
if (notDupIdx != nullptr) {
|
||
notDupIdx->insert(pBest->read1IndexInFile);
|
||
}
|
||
for (auto pe : vpRe) {
|
||
if (pe != pBest) {
|
||
dupIdx->insert(pe->read1IndexInFile);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/* 找到与readend pos相等的所有readend */
|
||
static void getEqualRE(const ReadEnds &re, vector<ReadEnds> &src, vector<ReadEnds> *dst) {
|
||
auto range = std::equal_range(src.begin(), src.end(), re, ReadEnds::PairsLittleThan); // 只比对位点
|
||
dst->insert(dst->end(), range.first, range.second);
|
||
}
|
||
|
||
/* 单线程生成readends (第一步)*/
|
||
static void generateReadEnds(MarkDupDataArg *arg) {
|
||
auto &p = *arg;
|
||
auto &rnParser = g_vRnParser[0];
|
||
|
||
p.pairs.clear();
|
||
p.frags.clear();
|
||
p.unpairedDic.clear();
|
||
p.unpairedPosArr.clear();
|
||
|
||
// bam_num1 += p.bams.size();
|
||
/* 处理每个read，创建ReadEnd，并放入frag和pair中 */
|
||
// MDSet<ReadEnds> reSet;
|
||
// ReadEnds lastRe;
|
||
|
||
for (size_t i = 0; i < p.bams.size(); ++i) { // 循环处理每个read
|
||
BamWrap *bw = p.bams[i];
|
||
// ++bam_num1;
|
||
const int64_t bamIdx = p.bamStartIdx + i;
|
||
if (bw->GetReadUnmappedFlag()) {
|
||
if (bw->b->core.tid == -1)
|
||
// When we hit the unmapped reads with no coordinate, no reason to continue (only in coordinate sort).
|
||
break;
|
||
} else if (!bw->IsSecondaryOrSupplementary()) { // 是主要比对
|
||
ReadEnds fragEnd;
|
||
tm_arr[8].acc_start();
|
||
buildReadEnds(*bw, bamIdx, rnParser, &fragEnd);
|
||
tm_arr[8].acc_end();
|
||
p.frags.push_back(fragEnd); // 添加进frag集合
|
||
if (bw->GetReadPairedFlag() && !bw->GetMateUnmappedFlag()) { // 是pairend而且互补的read也比对上了
|
||
string key = bw->query_name();
|
||
if (p.unpairedDic.find(key) == p.unpairedDic.end()) {
|
||
p.unpairedDic[key] = {p.taskSeq, fragEnd};
|
||
} else { // 找到了pairend
|
||
auto &pairedEnds = p.unpairedDic.at(key).unpairedRE;
|
||
modifyPairedEnds(fragEnd, &pairedEnds);
|
||
p.pairs.push_back(pairedEnds);
|
||
p.unpairedDic.erase(key); // 删除找到的pairend
|
||
}
|
||
}
|
||
}
|
||
}
|
||
tm_arr[9].acc_start();
|
||
sortReadEndsArr(p.frags);
|
||
// sort(p.frags.begin(), p.frags.end());
|
||
tm_arr[9].acc_end();
|
||
// cout << "sort pairs" << endl;
|
||
tm_arr[10].acc_start();
|
||
sort(p.pairs.begin(), p.pairs.end());
|
||
tm_arr[10].acc_end();
|
||
// 记录位点上的未匹配的read个数
|
||
for (auto &e : p.unpairedDic) {
|
||
auto posKey = e.second.unpairedRE.posKey;
|
||
auto &unpairArrInfo = p.unpairedPosArr[posKey];
|
||
unpairArrInfo.unpairedNum++;
|
||
unpairArrInfo.taskSeq = p.taskSeq;
|
||
unpairArrInfo.readNameSet.insert(e.first);
|
||
}
|
||
if (p.taskSeq == 98) {
|
||
for (auto &re : p.unpairedPosArr[14293757783047].pairArr) {
|
||
cout << "task 99 unpair reads: " << re.read1IndexInFile << "\t" << re.read2IndexInFile << endl;
|
||
}
|
||
}
|
||
// cout << "依赖比例：" << (float)p.unpairedDic.size() / p.frags.size() <<
|
||
// endl;
|
||
}
|
||
|
||
/* 处理pairs */
|
||
static void processPairs(vector<ReadEnds> &readEnds, DPSet<DupInfo> *dupIdx, MDSet<int64_t> *opticalDupIdx,
|
||
DPSet<DupInfo> *repIdx, MDSet<int64_t> *notDupIdx = nullptr,
|
||
MDSet<int64_t> *notOpticalDupIdx = nullptr, MDSet<int64_t> *notRepIdx = nullptr) {
|
||
// return;
|
||
vector<const ReadEnds *> vpCache; // 有可能是冗余的reads
|
||
const ReadEnds *pReadEnd = nullptr;
|
||
for (auto &re : readEnds) {
|
||
if (pReadEnd != nullptr && ReadEnds::AreComparableForDuplicates(*pReadEnd, re, true)) // 跟前一个一样
|
||
vpCache.push_back(&re); // 处理一个潜在的冗余组
|
||
else {
|
||
markDupsForPairs(vpCache, dupIdx, opticalDupIdx, repIdx, notDupIdx, notOpticalDupIdx, notRepIdx); // 不一样
|
||
vpCache.clear();
|
||
vpCache.push_back(&re);
|
||
pReadEnd = &re;
|
||
}
|
||
}
|
||
markDupsForPairs(vpCache, dupIdx, opticalDupIdx, repIdx, notDupIdx, notOpticalDupIdx, notRepIdx);
|
||
}
|
||
|
||
/* 处理frags */
|
||
static void processFrags(vector<ReadEnds> &readEnds, DPSet<DupInfo> *dupIdx, MDSet<int64_t> *notDupIdx = nullptr) {
|
||
bool containsPairs = false;
|
||
bool containsFrags = false;
|
||
vector<const ReadEnds *> vpCache; // 有可能是冗余的reads
|
||
const ReadEnds *pReadEnd = nullptr;
|
||
for (auto &re : readEnds) {
|
||
if (pReadEnd != nullptr && ReadEnds::AreComparableForDuplicates(*pReadEnd, re, false)) {
|
||
vpCache.push_back(&re);
|
||
containsPairs = containsPairs || re.IsPaired();
|
||
containsFrags = containsFrags || !re.IsPaired();
|
||
} else {
|
||
if (vpCache.size() > 1 && containsFrags) {
|
||
markDupsForFrags(vpCache, containsPairs, dupIdx, notDupIdx);
|
||
}
|
||
vpCache.clear();
|
||
vpCache.push_back(&re);
|
||
pReadEnd = &re;
|
||
containsPairs = re.IsPaired();
|
||
containsFrags = !re.IsPaired();
|
||
}
|
||
}
|
||
if (vpCache.size() > 1 && containsFrags) {
|
||
markDupsForFrags(vpCache, containsPairs, dupIdx, notDupIdx);
|
||
}
|
||
}
|
||
|
||
/* 单线程markdup (第二步)*/
|
||
static void markdups(MarkDupDataArg *arg) {
|
||
auto &p = *arg;
|
||
p.pairDupIdx.clear();
|
||
p.pairOpticalDupIdx.clear();
|
||
p.fragDupIdx.clear();
|
||
p.pairRepIdx.clear();
|
||
/* generateDuplicateIndexes，计算冗余read在所有read中的位置索引 */
|
||
// 先处理 pair
|
||
processPairs(p.pairs, &p.pairDupIdx, &p.pairOpticalDupIdx, &p.pairRepIdx);
|
||
// cout << p.pairDupIdx.size() << "\t" << p.pairRepIdx.size() << endl;
|
||
|
||
// 再处理frag
|
||
processFrags(p.frags, &p.fragDupIdx);
|
||
}
|
||
|
||
/* 获取交叉部分的数据 */
|
||
static inline void getIntersectData(vector<ReadEnds> &leftArr, vector<ReadEnds> &rightArr, vector<ReadEnds> *dst,
|
||
bool isPairCmp = false) {
|
||
if (leftArr.empty() || rightArr.empty()) {
|
||
// cout << "bad size: " << leftArr.size() << '\t' << rightArr.size() << endl;
|
||
return;
|
||
}
|
||
const size_t leftEndIdx = leftArr.size() - 1;
|
||
const size_t rightStartIdx = 0;
|
||
size_t leftSpan = 0;
|
||
size_t rightSpan = 0;
|
||
|
||
while (!ReadEnds::ReadLittleThan(leftArr[leftEndIdx - leftSpan], rightArr[rightStartIdx], isPairCmp)) {
|
||
leftSpan += 1;
|
||
if (leftSpan > leftEndIdx) { // 上一个的范围被下一个全部包围了（可能会有bug，上上个也与下一个有交集呢？）
|
||
leftSpan = leftArr.size() - 1;
|
||
break;
|
||
}
|
||
}
|
||
|
||
while (!ReadEnds::ReadLittleThan(leftArr[leftEndIdx], rightArr[rightSpan], isPairCmp)) {
|
||
rightSpan += 1;
|
||
if (rightSpan == rightArr.size() - 1) // 同上，可能会有bug
|
||
break;
|
||
}
|
||
dst->insert(dst->end(), leftArr.end() - leftSpan, leftArr.end());
|
||
dst->insert(dst->end(), rightArr.begin(), rightArr.begin() + rightSpan);
|
||
std::sort(dst->begin(), dst->end());
|
||
}
|
||
|
||
/* 将frags重叠部分的dup idx放进数据中 */
|
||
static inline void refreshFragDupIdx(DPSet<DupInfo> &dupIdx, MDSet<int64_t> ¬DupIdx, MarkDupDataArg *lastArg,
|
||
MarkDupDataArg *curArg) {
|
||
auto &lp = *lastArg;
|
||
auto &p = *curArg;
|
||
for (auto idx : dupIdx) {
|
||
lp.fragDupIdx.insert(idx);
|
||
p.fragDupIdx.erase(idx);
|
||
}
|
||
for (auto idx : notDupIdx) {
|
||
lp.fragDupIdx.erase(idx);
|
||
p.fragDupIdx.erase(idx);
|
||
}
|
||
}
|
||
|
||
/* 将pairs重叠部分的dup idx放进数据中 */
|
||
static inline void refreshPairDupIdx(DPSet<DupInfo> &dupIdx, MDSet<int64_t> &opticalDupIdx, DPSet<DupInfo> &repIdx,
|
||
MDSet<int64_t> ¬DupIdx, MDSet<int64_t> ¬OpticalDupIdx,
|
||
MDSet<int64_t> ¬RepIdx, MarkDupDataArg *lastArg, MarkDupDataArg *curArg) {
|
||
auto &lp = *lastArg;
|
||
auto &p = *curArg;
|
||
for (auto idx : dupIdx) {
|
||
lp.pairDupIdx.insert(idx);
|
||
p.pairDupIdx.erase(idx);
|
||
}
|
||
for (auto idx : opticalDupIdx) {
|
||
lp.pairOpticalDupIdx.insert(idx);
|
||
p.pairOpticalDupIdx.erase(idx);
|
||
}
|
||
for (auto idx : repIdx) {
|
||
lp.pairRepIdx.insert(idx);
|
||
p.pairRepIdx.erase(idx);
|
||
}
|
||
// for (auto idx : notDupIdx) {
|
||
// if (lp.pairDupIdx.find(idx) != lp.pairDupIdx.end()) cout << "find-1: " << idx << endl;
|
||
// if (lp.pairDupIdx.find({idx}) != lp.pairDupIdx.end()) cout << "find-2: " << idx << endl;
|
||
// if (p.pairDupIdx.find(idx) != p.pairDupIdx.end()) cout << "find-3: " << idx << endl;
|
||
// if (p.pairDupIdx.find({idx}) != p.pairDupIdx.end()) cout << "find-4: " << idx << endl;
|
||
// lp.pairDupIdx.erase(idx); p.pairDupIdx.erase(idx);
|
||
// }
|
||
for (auto idx : notOpticalDupIdx) {
|
||
lp.pairOpticalDupIdx.erase(idx);
|
||
p.pairOpticalDupIdx.erase(idx);
|
||
}
|
||
for (auto idx : notRepIdx) {
|
||
lp.pairRepIdx.erase(idx);
|
||
p.pairRepIdx.erase(idx);
|
||
}
|
||
}
|
||
|
||
// 用来分别处理dup和optical dup
|
||
static void refeshTaskDupInfo(DPSet<DupInfo> &dupIdx, MDSet<int64_t> &opticalDupIdx, DPSet<DupInfo> &repIdx,
|
||
MDSet<int64_t> ¬DupIdx, MDSet<int64_t> ¬OpticalDupIdx, MDSet<int64_t> ¬RepIdx,
|
||
DPSet<DupInfo> &latterDupIdx, MDSet<int64_t> &latterOpticalDupIdx,
|
||
DPSet<DupInfo> &latterRepIdx, MDSet<int64_t> &latterNotDupIdx,
|
||
MDSet<int64_t> &latterNotOpticalDupIdx, MDSet<int64_t> &latterNotRepIdx) {
|
||
for (auto idx : dupIdx) {
|
||
latterDupIdx.insert(idx);
|
||
// latterNotDupIdx.erase(idx); // 后来的更新为准
|
||
}
|
||
for (auto idx : opticalDupIdx) latterOpticalDupIdx.insert(idx);
|
||
for (auto idx : repIdx) latterRepIdx.insert(idx);
|
||
for (auto idx : notDupIdx) latterNotDupIdx.insert(idx);
|
||
for (auto idx : notOpticalDupIdx) latterNotOpticalDupIdx.insert(idx);
|
||
for (auto idx : notRepIdx) latterNotRepIdx.insert(idx);
|
||
}
|
||
|
||
/* 最后合并数据并排序 */
|
||
template <typename DupContainer, typename T>
|
||
static void refeshFinalTaskDupInfo(DupContainer &dupIdx, MDSet<int64_t> ¬DupIdx, vector<T> &dupArr,
|
||
vector<T> &cacheDupIdx, vector<T> &midArr) {
|
||
midArr.resize(0);
|
||
cacheDupIdx.resize(0);
|
||
cacheDupIdx.insert(cacheDupIdx.end(), dupIdx.begin(), dupIdx.end());
|
||
std::sort(cacheDupIdx.begin(), cacheDupIdx.end());
|
||
|
||
auto ai = dupArr.begin();
|
||
auto ae = dupArr.end();
|
||
auto bi = cacheDupIdx.begin();
|
||
auto be = cacheDupIdx.end();
|
||
// auto bi = dupIdx.begin();
|
||
// auto be = dupIdx.end();
|
||
|
||
T val = 0;
|
||
while (ai != ae && bi != be) {
|
||
if (*ai < *bi) {
|
||
val = *ai++;
|
||
} else if (*bi < *ai) {
|
||
val = *bi++;
|
||
} else {
|
||
val = *bi++; // 相等的时候取后面的作为结果
|
||
ai++;
|
||
}
|
||
if (notDupIdx.find(val) == notDupIdx.end()) {
|
||
midArr.push_back(val);
|
||
}
|
||
}
|
||
while (ai != ae) {
|
||
val = *ai++;
|
||
if (notDupIdx.find(val) == notDupIdx.end()) {
|
||
midArr.push_back(val);
|
||
}
|
||
}
|
||
while (bi != be) {
|
||
val = *bi++;
|
||
if (notDupIdx.find(val) == notDupIdx.end()) {
|
||
midArr.push_back(val);
|
||
}
|
||
}
|
||
dupArr = midArr;
|
||
}
|
||
|
||
/* 处理相邻的两个任务，有相交叉的数据 */
|
||
static void handleIntersectData(MarkDupDataArg *lastArg, MarkDupDataArg *curArg, GlobalDataArg *gDataArg) {
|
||
auto &lp = *lastArg;
|
||
auto &p = *curArg;
|
||
auto &g = *gDataArg;
|
||
|
||
vector<ReadEnds> reArr;
|
||
DPSet<DupInfo> dupIdx;
|
||
MDSet<int64_t> opticalDupIdx;
|
||
DPSet<DupInfo> repIdx;
|
||
MDSet<int64_t> notOpticalDupIdx;
|
||
MDSet<int64_t> notDupIdx;
|
||
MDSet<int64_t> notRepIdx;
|
||
// 先处理重叠的frags
|
||
getIntersectData(lp.frags, p.frags, &reArr);
|
||
processFrags(reArr, &dupIdx, ¬DupIdx);
|
||
refreshFragDupIdx(dupIdx, notDupIdx, &lp, &p);
|
||
|
||
// 再处理重叠的pairs
|
||
reArr.clear();
|
||
dupIdx.clear();
|
||
notDupIdx.clear();
|
||
getIntersectData(lp.pairs, p.pairs, &reArr, true);
|
||
processPairs(reArr, &dupIdx, &opticalDupIdx, &repIdx, ¬DupIdx, ¬OpticalDupIdx, ¬RepIdx);
|
||
refreshPairDupIdx(dupIdx, opticalDupIdx, repIdx, notDupIdx, notOpticalDupIdx, notRepIdx, &lp, &p);
|
||
|
||
// cout << (g.unpairedDic.find("A01415:368:HL7NTDSX3:3:1104:5195:34757") != g.unpairedDic.end()) << endl;
|
||
// cout << (g.unpairedPosArr.find(14293757783047) != g.unpairedPosArr.end()) << endl;
|
||
// 处理之前未匹配的部分
|
||
map<CalcKey, int64_t> recalcPos;
|
||
CalcSet<CalcKey> alreadyAdd; // 与该位点相同的pair都添加到数组里了
|
||
MDSet<int64_t> addToGlobal;
|
||
int64_t prevLastPos = 0, nextFirstPos = 0;
|
||
if (lp.frags.size() > 0)
|
||
prevLastPos = lp.frags.back().posKey;
|
||
if (p.frags.size() > 0)
|
||
nextFirstPos = p.frags[0].posKey;
|
||
// cout << "range: " << nextFirstPos << '\t' << prevLastPos << endl;
|
||
for (auto &prevUnpair : lp.unpairedDic) { // 遍历上一个任务中的每个未匹配的read
|
||
auto &readName = prevUnpair.first;
|
||
// if (readName == "A01415:368:HL7NTDSX3:3:1104:5195:34757" ||
|
||
// readName == "A01415:368:HL7NTDSX3:3:1104:4887:32095") {
|
||
// cout << "read name found: " << lp.taskSeq << endl;
|
||
// }
|
||
auto &prevPosInfo = prevUnpair.second;
|
||
auto prevFragEnd = prevPosInfo.unpairedRE; // 未匹配的read end
|
||
// if (prevFragEnd.read1IndexInFile == 255830545 || prevFragEnd.read1IndexInFile == 255830546
|
||
// || prevFragEnd.read1IndexInFile == 255832599 || prevFragEnd.read1IndexInFile == 255832601) {
|
||
// cout << prevFragEnd.read1IndexInFile << "\t" << readName << endl;
|
||
// }
|
||
|
||
if (p.unpairedDic.find(readName) != p.unpairedDic.end()) { // 在当前这个任务里找到了这个未匹配的read
|
||
auto &nextPosInfo = p.unpairedDic[readName];
|
||
auto &nextFragEnd = nextPosInfo.unpairedRE;
|
||
int64_t prevPosKey = prevFragEnd.posKey;
|
||
modifyPairedEnds(nextFragEnd, &prevFragEnd); // 在某些clip情况下，poskey可能是后面的read
|
||
int64_t nextPosKey = max(prevPosKey, nextFragEnd.posKey);
|
||
CalcKey ck = {prevPosKey, nextPosKey};
|
||
UnpairedPosInfo *prevUnpairInfoP = nullptr;
|
||
UnpairedPosInfo *nextUnpairInfoP = nullptr;
|
||
if (lp.unpairedPosArr.find(prevPosKey) != lp.unpairedPosArr.end())
|
||
prevUnpairInfoP = &lp.unpairedPosArr[prevPosKey];
|
||
if (p.unpairedPosArr.find(prevPosKey) != p.unpairedPosArr.end())
|
||
nextUnpairInfoP = &p.unpairedPosArr[prevPosKey];
|
||
|
||
// if (prevFragEnd.read1IndexInFile == 255830545 || prevFragEnd.read1IndexInFile == 255830546 ||
|
||
// prevFragEnd.read1IndexInFile == 255832599 || prevFragEnd.read1IndexInFile == 255832601) {
|
||
// cout << "find in p: " << lp.taskSeq << "\t" << prevFragEnd.read1IndexInFile << "\t" << readName <<
|
||
// endl; if (nextUnpairInfoP != nullptr)
|
||
// cout << "next p: " << nextUnpairInfoP->unpairedNum << endl;
|
||
// if (prevUnpairInfoP != nullptr)
|
||
// cout << "prev p: " << prevUnpairInfoP->unpairedNum << endl;
|
||
// cout << (g.unpairedDic.find(readName) != g.unpairedDic.end()) << endl;
|
||
// }
|
||
|
||
// pos分为两种情况，根据poskey(pair中两个read分别的pos)的位置确定
|
||
// 1.
|
||
// prevpos在交叉部分之前，nextpos在交叉部分之后，这种情况不需要获取pairarr中的数据;
|
||
// 2.
|
||
// prevpos在交叉部分之前，nextpos在交叉部分，需要获取lp中的相等read pair进行重新计算
|
||
// 复杂情况1. g中包含prevPosKey对应的unpair，p中有对应的pair，此时应该把这些pair考虑进去
|
||
// 3.
|
||
// prevpos在交叉部分，nextpos在交叉部分之后，需要获取p中的相等read pair进行重新计算
|
||
// 复杂情况2. p中是否包含prevPosKey对应的unpair
|
||
// 4.
|
||
// prevpos在交叉部分，nextpos在交叉部分，需要获取lp和p中的相等read pair进行重新计算
|
||
|
||
bool addDataToPos = true;
|
||
if (alreadyAdd.find(ck) != alreadyAdd.end()) {
|
||
// 之前已经添加过了，后面就不用再添加数据了，因为同一个位置可能找到两个及以上的unpair数据，处理之前的数据时候可能已经添加了这些数据
|
||
addDataToPos = false;
|
||
} else
|
||
alreadyAdd.insert(ck);
|
||
|
||
if (prevPosKey < nextFirstPos) { // prevpos在交叉部分之前
|
||
auto &prevPairArr = prevUnpairInfoP->pairArr; // prevUnpairInfoP肯定不是nullptr
|
||
prevPairArr.push_back(prevFragEnd);
|
||
if (nextPosKey <= prevLastPos && addDataToPos) { // 第二种情况
|
||
getEqualRE(prevFragEnd, lp.pairs, &prevPairArr);
|
||
}
|
||
// 第一种情况，第二种情况下都会出现，复杂情况一
|
||
auto gPosInfo = g.unpairedPosArr.find(prevPosKey);
|
||
if (gPosInfo != g.unpairedPosArr.end()) { // 可能g和p有匹配的，刚好和该位点一致
|
||
auto &gUnpairInfo = gPosInfo->second;
|
||
auto pPosInfo = p.unpairedPosArr.find(nextPosKey);
|
||
if (pPosInfo != p.unpairedPosArr.end()) {
|
||
auto &pUnpairInfo = pPosInfo->second;
|
||
for (auto &rn : gUnpairInfo.readNameSet) { // 遍历每一个readname，看是否有匹配的
|
||
if (pUnpairInfo.readNameSet.find(rn) != pUnpairInfo.readNameSet.end()) {
|
||
auto pe = g.unpairedDic[rn].unpairedRE;
|
||
auto fe = p.unpairedDic[rn].unpairedRE;
|
||
modifyPairedEnds(fe, &pe);
|
||
prevPairArr.push_back(pe);
|
||
g.unpairedDic.erase(rn);
|
||
p.unpairedDic.erase(rn);
|
||
// cout << "找到了！" << rn << endl;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
recalcPos[ck] = prevPosInfo.taskSeq;
|
||
std::sort(prevPairArr.begin(), prevPairArr.end());
|
||
} else { // prevpos在交叉部分
|
||
if (nextPosKey > prevLastPos) { // nextpos在交叉部分之后 第三种情况
|
||
if (nextUnpairInfoP != nullptr) { // 且在pos点，next task有unpair，这样才把这些数据放到next task里
|
||
auto &nextPairArr = nextUnpairInfoP->pairArr;
|
||
nextPairArr.push_back(prevFragEnd);
|
||
auto &prevPairArr = prevUnpairInfoP->pairArr;
|
||
prevPairArr.push_back(prevFragEnd);
|
||
if (addDataToPos) {
|
||
getEqualRE(prevFragEnd, p.pairs, &prevPairArr);
|
||
getEqualRE(prevFragEnd, p.pairs, &nextPairArr);
|
||
}
|
||
// 将数据放到next task里,（这个位点以后会可能还会计算到，目前方案是都计算，只是把冗余剔除）
|
||
recalcPos[ck] = nextPosInfo.taskSeq;
|
||
|
||
std::sort(prevPairArr.begin(), prevPairArr.end());
|
||
std::sort(nextPairArr.begin(), nextPairArr.end());
|
||
} else { // next task在该位点没有unpair，那就把数据放到prev task里
|
||
auto &prevPairArr = prevUnpairInfoP->pairArr; // prevUnpairInfoP肯定不是nullptr
|
||
prevPairArr.push_back(prevFragEnd);
|
||
if (addDataToPos) // 第二种情况
|
||
getEqualRE(prevFragEnd, p.pairs, &prevPairArr);
|
||
recalcPos[ck] = prevPosInfo.taskSeq;
|
||
std::sort(prevPairArr.begin(), prevPairArr.end());
|
||
}
|
||
} else { // 第四种情况
|
||
if (prevUnpairInfoP == nullptr) {
|
||
prevUnpairInfoP = &lp.unpairedPosArr[prevPosKey];
|
||
prevUnpairInfoP->taskSeq = lp.taskSeq;
|
||
}
|
||
auto &prevPairArr = prevUnpairInfoP->pairArr;
|
||
prevPairArr.push_back(prevFragEnd);
|
||
if (addDataToPos) {
|
||
getEqualRE(prevFragEnd, lp.pairs, &prevPairArr);
|
||
getEqualRE(prevFragEnd, p.pairs, &prevPairArr);
|
||
}
|
||
recalcPos[ck] = prevPosInfo.taskSeq;
|
||
std::sort(prevPairArr.begin(), prevPairArr.end());
|
||
}
|
||
}
|
||
p.unpairedDic.erase(readName); // 在next task里删除该read
|
||
} else if (g.unpairedDic.find(readName) != g.unpairedDic.end()) { // 在遗留数据中找到了匹配的read
|
||
// if (readName == "A01415:368:HL7NTDSX3:3:1104:5195:34757" ||
|
||
// readName == "A01415:368:HL7NTDSX3:3:1104:4887:32095") {
|
||
// cout << "find in g: " << prevFragEnd.read1IndexInFile << "\t" << readName << endl;
|
||
// }
|
||
auto &remainPosInfo = g.unpairedDic[readName];
|
||
auto remainFragEnd = remainPosInfo.unpairedRE;
|
||
int64_t remainPosKey = remainFragEnd.posKey;
|
||
modifyPairedEnds(prevFragEnd, &remainFragEnd); // 在某些clip情况下，poskey可能是后面的read
|
||
auto &remainUnpairInfo = g.unpairedPosArr[remainPosKey];
|
||
auto &remainPairArr = remainUnpairInfo.pairArr;
|
||
remainPairArr.push_back(remainFragEnd);
|
||
CalcKey ck = {remainPosKey, prevFragEnd.posKey};
|
||
recalcPos[ck] = remainPosInfo.taskSeq;
|
||
std::sort(remainPairArr.begin(), remainPairArr.end());
|
||
|
||
g.unpairedDic.erase(readName);
|
||
} else { // 都没找到，那就保存到遗留数据里
|
||
int64_t prevPosKey = prevFragEnd.posKey;
|
||
g.unpairedDic.insert(prevUnpair);
|
||
addToGlobal.insert(prevPosKey);
|
||
// if (prevFragEnd.read1IndexInFile == 255830545 || prevFragEnd.read1IndexInFile == 255830546) {
|
||
// cout << "here find" << endl;
|
||
// }
|
||
}
|
||
}
|
||
|
||
map<int64_t, TaskSeqDupInfo> taskChanged;
|
||
MDSet<int64_t> posProcessed;
|
||
for (auto &e : recalcPos) {
|
||
auto posKey = e.first.read1Pos;
|
||
if (posProcessed.find(posKey) != posProcessed.end())
|
||
continue;
|
||
posProcessed.insert(posKey);
|
||
auto taskSeq = e.second;
|
||
auto &t = taskChanged[taskSeq];
|
||
// 在对应的任务包含的dup idx里修改结果数据
|
||
vector<ReadEnds> *pairArrP = nullptr;
|
||
if (taskSeq < lp.taskSeq)
|
||
pairArrP = &g.unpairedPosArr[posKey].pairArr;
|
||
else
|
||
pairArrP = &lp.unpairedPosArr[posKey].pairArr;
|
||
// if (taskSeq == 98) cout << "handle recalc pairs: " << taskSeq << "\t" << posKey << endl;
|
||
// if (p.taskSeq == 163) {
|
||
// cout << "final" << endl;
|
||
// }
|
||
processPairs(*pairArrP, &t.dupIdx, &t.opticalDupIdx, &t.repIdx, &t.notDupIdx, &t.notOpticalDupIdx,
|
||
&t.notRepIdx);
|
||
if (taskSeq < lp.taskSeq)
|
||
g.unpairedPosArr.erase(posKey);
|
||
}
|
||
|
||
// if (lp.taskSeq > 98) {
|
||
// // exit(0);
|
||
// }
|
||
|
||
// 最后再添加，以防开始赋值，后来这个位置要是又添加了新的数据
|
||
// 放在这里，因为lp中的unpairedPosArr中的readends可能会被修改（比如optical duplicate）
|
||
for (auto posKey : addToGlobal) {
|
||
// if (posKey == 14293757783047) {
|
||
// for (auto &re: lp.unpairedPosArr[posKey].pairArr) {
|
||
// cout << "lp reads: " << re.read1IndexInFile << "\t" << re.read2IndexInFile << endl;
|
||
// }
|
||
// cout << "found in g: " << lp.taskSeq << "\t" << lp.unpairedPosArr[posKey].unpairedNum << "\t" <<
|
||
// lp.unpairedPosArr[posKey].pairArr.size() << endl;
|
||
// }
|
||
g.unpairedPosArr[posKey] = lp.unpairedPosArr[posKey];
|
||
}
|
||
|
||
// 更新结果
|
||
for (auto &e : taskChanged) {
|
||
auto taskSeq = e.first;
|
||
auto &t = e.second;
|
||
// cout << t.dupIdx.size() << "\t" << t.notDupIdx.size() << endl;
|
||
if (taskSeq < lp.taskSeq) {
|
||
refeshTaskDupInfo(t.dupIdx, t.opticalDupIdx, t.repIdx, t.notDupIdx, t.notOpticalDupIdx, t.notRepIdx,
|
||
g.latterDupIdxArr[taskSeq], g.latterOpticalDupIdxArr[taskSeq], g.latterRepIdxArr[taskSeq],
|
||
g.latterNotDupIdxArr[taskSeq], g.latterNotOpticalDupIdxArr[taskSeq],
|
||
g.latterNotRepIdxArr[taskSeq]);
|
||
} else if (taskSeq == lp.taskSeq) {
|
||
refreshPairDupIdx(t.dupIdx, t.opticalDupIdx, t.repIdx, t.notDupIdx, t.notOpticalDupIdx, t.notRepIdx, &lp,
|
||
&p);
|
||
} else {
|
||
refreshPairDupIdx(t.dupIdx, t.opticalDupIdx, t.repIdx, t.notDupIdx, t.notOpticalDupIdx, t.notRepIdx, &p,
|
||
&lp); // 把结果放到p中
|
||
}
|
||
}
|
||
|
||
// cout << "remain unpaired: " << g.unpairedDic.size() << '\t' <<
|
||
// g.unpairedPosArr.size() << endl; cout << "calc g time: " <<
|
||
// t.seconds_elapsed() << " s" << endl; 将dupidx放进全局数据
|
||
g.latterDupIdxArr.push_back(DPSet<DupInfo>());
|
||
g.latterOpticalDupIdxArr.push_back(MDSet<int64_t>());
|
||
g.latterRepIdxArr.push_back(DPSet<DupInfo>());
|
||
g.latterNotDupIdxArr.push_back(MDSet<int64_t>());
|
||
g.latterNotOpticalDupIdxArr.push_back(MDSet<int64_t>());
|
||
g.latterNotRepIdxArr.push_back(MDSet<int64_t>());
|
||
|
||
g.dupIdxArr.push_back(vector<DupInfo>());
|
||
auto &vIdx = g.dupIdxArr.back();
|
||
lp.pairDupIdx.insert(lp.fragDupIdx.begin(), lp.fragDupIdx.end());
|
||
vIdx.insert(vIdx.end(), lp.pairDupIdx.begin(), lp.pairDupIdx.end());
|
||
std::sort(vIdx.begin(), vIdx.end());
|
||
// cout << vIdx.size() << endl;
|
||
// zzhtestnum += vIdx.size();
|
||
|
||
g.opticalDupIdxArr.push_back(vector<int64_t>());
|
||
auto &vOpticalIdx = g.opticalDupIdxArr.back();
|
||
vOpticalIdx.insert(vOpticalIdx.end(), lp.pairOpticalDupIdx.begin(), lp.pairOpticalDupIdx.end());
|
||
std::sort(vOpticalIdx.begin(), vOpticalIdx.end());
|
||
|
||
g.repIdxArr.push_back(vector<DupInfo>());
|
||
auto &vRepIdx = g.repIdxArr.back();
|
||
vRepIdx.insert(vRepIdx.end(), lp.pairRepIdx.begin(), lp.pairRepIdx.end());
|
||
std::sort(vRepIdx.begin(), vRepIdx.end());
|
||
}
|
||
|
||
/* 当所有任务结束后，global data里还有未处理的数据 */
|
||
static void handleLastTask(MarkDupDataArg *task, GlobalDataArg *gDataArg) {
|
||
// cout << "last task start" << endl;
|
||
auto &lp = *task;
|
||
auto &g = *gDataArg;
|
||
// 遗留的未匹配的pair
|
||
for (auto &prevUnpair : lp.unpairedDic) { // 遍历上一个任务中的每个未匹配的read
|
||
auto &readName = prevUnpair.first;
|
||
auto &prevPosInfo = prevUnpair.second;
|
||
auto prevFragEnd = prevPosInfo.unpairedRE; // 未匹配的read end
|
||
|
||
if (g.unpairedDic.find(readName) != g.unpairedDic.end()) { // 在遗留数据中找到了匹配的read
|
||
auto &remainPosInfo = g.unpairedDic[readName];
|
||
auto remainFragEnd = remainPosInfo.unpairedRE;
|
||
int64_t remainPosKey = remainFragEnd.posKey;
|
||
modifyPairedEnds(prevFragEnd, &remainFragEnd); // 在某些clip情况下，poskey可能是后面的read
|
||
auto &remainUnpairInfo = g.unpairedPosArr[remainPosKey];
|
||
|
||
remainUnpairInfo.pairArr.push_back(remainFragEnd);
|
||
g.unpairedDic.erase(readName);
|
||
} else {
|
||
g.unpairedDic.insert(prevUnpair); // 用来记录没有匹配的read个数
|
||
}
|
||
}
|
||
|
||
map<int64_t, TaskSeqDupInfo> taskChanged;
|
||
for (auto &e : g.unpairedPosArr) {
|
||
auto posKey = e.first;
|
||
auto taskSeq = e.second.taskSeq;
|
||
auto &t = taskChanged[taskSeq];
|
||
auto &arr = g.unpairedPosArr[posKey].pairArr;
|
||
|
||
if (arr.size() > 1) {
|
||
std::sort(arr.begin(), arr.end());
|
||
// cout << "last task before mark pair" << endl;
|
||
processPairs(arr, &t.dupIdx, &t.opticalDupIdx, &t.repIdx, &t.notDupIdx, &t.notRepIdx);
|
||
} else if (arr.size() == 1) {
|
||
gMetrics.singletonReads.insert(arr[0].read1IndexInFile);
|
||
}
|
||
}
|
||
// 更新结果
|
||
vector<int64_t> addDup;
|
||
map<int64_t, int64_t> ndPosVal;
|
||
for (auto &e : taskChanged) {
|
||
auto taskSeq = e.first;
|
||
auto &t = e.second;
|
||
// cout << t.dupIdx.size() << "\t" << t.notDupIdx.size() << endl;
|
||
refeshTaskDupInfo(t.dupIdx, t.opticalDupIdx, t.repIdx, t.notDupIdx, t.notOpticalDupIdx, t.notRepIdx,
|
||
g.latterDupIdxArr[taskSeq], g.latterOpticalDupIdxArr[taskSeq], g.latterRepIdxArr[taskSeq],
|
||
g.latterNotDupIdxArr[taskSeq], g.latterNotOpticalDupIdxArr[taskSeq],
|
||
g.latterNotRepIdxArr[taskSeq]);
|
||
}
|
||
|
||
// cout << "last unpair info: " << g.unpairedDic.size() << '\t' << g.unpairedPosArr.size() << endl;
|
||
g.unpairedPosArr.clear();
|
||
|
||
// for (auto &rn : g.unpairedDic) {
|
||
// cout << rn.first << endl;
|
||
// }
|
||
|
||
/*
|
||
int taskSeq = 0;
|
||
for (auto &arr : g.dupIdxArr) {
|
||
cout << taskSeq << "\t" << arr.size();
|
||
if (taskSeq < (int)g.dupIdxArr.size() - 1)
|
||
cout << "\t" << g.latterDupIdxArr[taskSeq].size() << "\t" << g.latterNotDupIdxArr[taskSeq].size() <<
|
||
endl; else cout << endl;
|
||
// if (taskSeq == 98) {
|
||
// vector<DupInfo> v;
|
||
// v.insert(v.end(), g.latterDupIdxArr[taskSeq].begin(), g.latterDupIdxArr[taskSeq].end());
|
||
// std::sort(v.begin(), v.end());
|
||
// for (auto &idx : v)
|
||
// cout << idx.idx << " " << idx.repIdx << " " << idx.dupSet << endl;
|
||
// v.clear();
|
||
// v.insert(v.end(), g.latterNotDupIdxArr[taskSeq].begin(), g.latterNotDupIdxArr[taskSeq].end());
|
||
// std::sort(v.begin(), v.end());
|
||
// for (auto &idx : v) cout << idx << endl;
|
||
// }
|
||
taskSeq++;
|
||
}
|
||
*/
|
||
// 将dupidx放进全局数据
|
||
vector<DupInfo> cacheDupIdx;
|
||
vector<DupInfo> midArr;
|
||
vector<int64_t> intCacheDupIdx;
|
||
vector<int64_t> intMidArr;
|
||
for (int i = 0; i < (int)g.dupIdxArr.size() - 1; ++i) {
|
||
/*
|
||
if (i == 98) {
|
||
cout << "98: " << g.latterDupIdxArr[i].size() << "\t" << g.dupIdxArr[i].size() << endl;
|
||
int inlatter = 0;
|
||
int innotdup = 0;
|
||
int latterinnotdup = 0;
|
||
for (auto idx : g.dupIdxArr[i]) {
|
||
if (g.latterDupIdxArr[i].find(idx) != g.latterDupIdxArr[i].end()) {
|
||
++inlatter;
|
||
}
|
||
}
|
||
for (auto idx : g.dupIdxArr[i]) {
|
||
if (g.latterNotDupIdxArr[i].find(idx) != g.latterNotDupIdxArr[i].end()) {
|
||
cout << idx.idx << endl;
|
||
++innotdup;
|
||
}
|
||
}
|
||
for (auto idx : g.latterDupIdxArr[i]) {
|
||
if (g.latterNotDupIdxArr[i].find(idx) != g.latterNotDupIdxArr[i].end()) {
|
||
++latterinnotdup;
|
||
}
|
||
}
|
||
cout << inlatter << "\t" << innotdup << "\t" << latterinnotdup << endl;
|
||
}
|
||
*/
|
||
refeshFinalTaskDupInfo(g.latterDupIdxArr[i], g.latterNotDupIdxArr[i], g.dupIdxArr[i], cacheDupIdx, midArr);
|
||
/*
|
||
if (i == 98) {
|
||
cout << "98: " << g.latterDupIdxArr[i].size() << "\t" << g.dupIdxArr[i].size() << endl;
|
||
}
|
||
*/
|
||
}
|
||
for (int i = 0; i < (int)g.opticalDupIdxArr.size() - 1; ++i)
|
||
refeshFinalTaskDupInfo(g.latterOpticalDupIdxArr[i], g.latterNotOpticalDupIdxArr[i], g.opticalDupIdxArr[i],
|
||
intCacheDupIdx, intMidArr);
|
||
for (int i = 0; i < (int)g.repIdxArr.size() - 1; ++i)
|
||
refeshFinalTaskDupInfo(g.latterRepIdxArr[i], g.latterNotRepIdxArr[i], g.repIdxArr[i], cacheDupIdx, midArr);
|
||
|
||
g.dupIdxArr.push_back(vector<DupInfo>());
|
||
auto &vIdx = g.dupIdxArr.back();
|
||
lp.pairDupIdx.insert(lp.fragDupIdx.begin(), lp.fragDupIdx.end());
|
||
vIdx.insert(vIdx.end(), lp.pairDupIdx.begin(), lp.pairDupIdx.end());
|
||
std::sort(vIdx.begin(), vIdx.end());
|
||
// cout << "last " << vIdx.size() << endl;
|
||
// zzhtestnum += vIdx.size();
|
||
/*
|
||
for (auto &arr : g.dupIdxArr) {
|
||
cout << taskSeq << "\t" << arr.size() << endl;
|
||
taskSeq++;
|
||
}
|
||
*/
|
||
|
||
g.opticalDupIdxArr.push_back(vector<int64_t>());
|
||
auto &vOpticalIdx = g.opticalDupIdxArr.back();
|
||
vOpticalIdx.insert(vOpticalIdx.end(), lp.pairOpticalDupIdx.begin(), lp.pairOpticalDupIdx.end());
|
||
std::sort(vOpticalIdx.begin(), vOpticalIdx.end());
|
||
|
||
g.repIdxArr.push_back(vector<DupInfo>());
|
||
auto &vRepIdx = g.repIdxArr.back();
|
||
vRepIdx.insert(vRepIdx.end(), lp.pairRepIdx.begin(), lp.pairRepIdx.end());
|
||
std::sort(vRepIdx.begin(), vRepIdx.end());
|
||
}
|
||
|
||
void calculateMetrics(MarkDupDataArg &lp, MarkDupDataArg &p, GlobalDataArg &g, DuplicationMetrics *pgMetrics) {
|
||
DuplicationMetrics &gMetrics = *pgMetrics;
|
||
|
||
// gMetrics.READ_PAIRS_EXAMINED /= 2;
|
||
|
||
cout << "calculateMetrics start: " << endl;
|
||
|
||
cout << lp.unpairedDic.size() << "\t" << p.unpairedDic.size() << "\t" << g.unpairedDic.size() << endl;
|
||
cout << "all: " << (lp.unpairedDic.size() + p.unpairedDic.size() + g.unpairedDic.size()) << endl;
|
||
cout << "calculateMetrics end" << endl;
|
||
}
|
||
|
||
/* 串行处理数据，标记冗余 */
|
||
void serialMarkDups() {
|
||
tm_arr[5].acc_start();
|
||
Timer::log_time("serial start");
|
||
// 读取缓存初始化
|
||
BamBufType inBamBuf(g_gArg.use_asyncio);
|
||
inBamBuf.Init(g_inBamFp, g_inBamHeader, g_gArg.max_mem);
|
||
// BamBufType inBamBuf(false);
|
||
// inBamBuf.Init(g_inBamFp, g_inBamHeader, 100 * 1024 * 1024);
|
||
int64_t processedBamNum = 0;
|
||
|
||
MarkDupDataArg smdArg1, smdArg2;
|
||
MarkDupDataArg *lastArgP = &smdArg1;
|
||
MarkDupDataArg *curArgP = &smdArg2;
|
||
|
||
bool isFirstRound = true;
|
||
int roundNum = 0;
|
||
int64_t readNumSum = 0;
|
||
while (inBamBuf.ReadStat() >= 0) {
|
||
Timer t_round;
|
||
// 读取bam文件中的read
|
||
tm_arr[4].acc_start();
|
||
size_t readNum = inBamBuf.ReadBam();
|
||
readNumSum += readNum;
|
||
// readNumSum += inBamBuf.GetBamArr().size();
|
||
tm_arr[4].acc_end();
|
||
cout << "read num: " << readNum << '\t' << roundNum << endl;
|
||
// lastArgP = curArgP;
|
||
tm_arr[6].acc_start();
|
||
curArgP->taskSeq = roundNum;
|
||
curArgP->bamStartIdx = processedBamNum;
|
||
curArgP->bams = inBamBuf.GetBamArr();
|
||
tm_arr[6].acc_end();
|
||
|
||
tm_arr[0].acc_start();
|
||
Timer t1;
|
||
generateReadEnds(curArgP);
|
||
// cout << "calc read end time: " << t1.seconds_elapsed() << " s" <<
|
||
// endl;
|
||
tm_arr[0].acc_end();
|
||
|
||
tm_arr[1].acc_start();
|
||
t1.reinit();
|
||
markdups(curArgP);
|
||
// cout << "markdups time: " << t1.seconds_elapsed() << " s" << endl;
|
||
tm_arr[1].acc_end();
|
||
|
||
if (!isFirstRound) {
|
||
tm_arr[2].acc_start();
|
||
t1.reinit();
|
||
handleIntersectData(lastArgP, curArgP, &gData);
|
||
// cout << "intersect time: " << t1.seconds_elapsed() << " s" <<
|
||
// endl;
|
||
// addTaskIdxToSet(lastArgP, &gData);
|
||
tm_arr[2].acc_end();
|
||
} else {
|
||
isFirstRound = false;
|
||
}
|
||
inBamBuf.ClearAll(); // 清理上一轮读入的数据
|
||
processedBamNum += readNum;
|
||
|
||
// 交换
|
||
auto tmp = lastArgP;
|
||
lastArgP = curArgP;
|
||
curArgP = tmp;
|
||
// cout << "round time: " << t_round.seconds_elapsed() << endl;
|
||
roundNum++;
|
||
if (roundNum % 100 == 0) {
|
||
// cout << "read sum: " << readNumSum << endl;
|
||
// cout << "round time: " << t_round.seconds_elapsed() * 100 << " s" << endl;
|
||
}
|
||
}
|
||
// cout << "here" << endl;
|
||
tm_arr[3].acc_start();
|
||
// 处理剩下的全局数据
|
||
handleLastTask(lastArgP, &gData);
|
||
|
||
// 计算各种统计指标
|
||
// calculateMetrics(*lastArgP, *curArgP, gData, &gMetrics);
|
||
|
||
// cout << "here 2" << endl;
|
||
tm_arr[3].acc_end();
|
||
|
||
tm_arr[5].acc_end();
|
||
// 统计所有冗余index数量
|
||
int64_t dupNum = 0;
|
||
int64_t opticalDupNum = 0;
|
||
|
||
map<int64_t, int> dup;
|
||
|
||
int taskSeq = 0;
|
||
/* for (auto &arr : gData.dupIdxArr) {
|
||
for (auto idx : arr) {
|
||
if (dup.find(idx.idx) != dup.end()) {
|
||
// cout << "dup index: " << dup[idx] << '\t' << taskSeq << '\t'
|
||
// << idx << endl;
|
||
}
|
||
dup[idx.idx] = taskSeq;
|
||
}
|
||
// cout << taskSeq << "\t" << arr.size() << endl;
|
||
taskSeq++;
|
||
}
|
||
*/
|
||
// #include <fstream>
|
||
// ofstream out("tumor_dup.txt");
|
||
// for (auto idx : dup)
|
||
// {
|
||
// out << idx << endl;
|
||
// }
|
||
// out.close();
|
||
|
||
for (auto &arr : gData.dupIdxArr) dupNum += arr.size();
|
||
for (auto &arr : gData.opticalDupIdxArr) opticalDupNum += arr.size();
|
||
|
||
cout << "dup num : " << dupNum << '\t' << dup.size() << "\t" << zzhtestnum << endl;
|
||
|
||
cout << "calc readend: " << tm_arr[0].acc_seconds_elapsed() << endl;
|
||
cout << "markdup : " << tm_arr[1].acc_seconds_elapsed() << endl;
|
||
cout << "handle tail : " << tm_arr[2].acc_seconds_elapsed() << endl;
|
||
cout << "handle last : " << tm_arr[3].acc_seconds_elapsed() << endl;
|
||
cout << "read bam : " << tm_arr[4].acc_seconds_elapsed() << endl;
|
||
cout << "new arg : " << tm_arr[6].acc_seconds_elapsed() << endl;
|
||
cout << "del arg : " << tm_arr[7].acc_seconds_elapsed() << endl;
|
||
cout << "build ends : " << tm_arr[8].acc_seconds_elapsed() << endl;
|
||
cout << "sort frags : " << tm_arr[9].acc_seconds_elapsed() << endl;
|
||
cout << "sort pairs : " << tm_arr[10].acc_seconds_elapsed() << endl;
|
||
cout << "all : " << tm_arr[5].acc_seconds_elapsed() << endl;
|
||
cout << "optical size: " << opticalDupNum << endl;
|
||
cout << "singleton size: " << gMetrics.singletonReads.size() << endl;
|
||
|
||
Timer::log_time("serial end ");
|
||
cout << "read num sum: " << readNumSum << endl;
|
||
|
||
// for (auto i : gData.dupArr)
|
||
// cout << i << endl;
|
||
} |