96 lines
2.9 KiB
C++
96 lines
2.9 KiB
C++
/*
|
||
Description: 第一阶段写入中间文件
|
||
|
||
Copyright : All right reserved by ICT
|
||
|
||
Author : Zhang Zhonghai
|
||
Date : 2026/06/02
|
||
*/
|
||
|
||
#include "phase_1_write.h"
|
||
|
||
#include <klib/kthread.h>
|
||
#include <spdlog/spdlog.h>
|
||
#include <zlib.h>
|
||
|
||
#include <algorithm>
|
||
#include <string>
|
||
|
||
#include "common_data.h"
|
||
#include "const_val.h"
|
||
#include "phase_1.h"
|
||
#include "phase_1_compress.h"
|
||
#include "phase_1_write.h"
|
||
#include "sort.h"
|
||
#include "util/profiling.h"
|
||
|
||
static void checkCompress(uint8_t *addr, uint64_t len) {
|
||
size_t curReadPos = 0;
|
||
int blockLen = 0;
|
||
int maxBlockLen = 0;
|
||
DataBuffer buf;
|
||
buf.AllocMem(SINGLE_BLOCK_SIZE);
|
||
|
||
while (curReadPos + BLOCK_HEADER_LENGTH <= len) { /* 确保能解析block长度 */
|
||
blockLen = unpackInt16(&addr[curReadPos + 16]) + 1;
|
||
if (blockLen > maxBlockLen) {
|
||
maxBlockLen = blockLen;
|
||
}
|
||
if (curReadPos + blockLen <= len) { /* 完整的block数据在buf里 */
|
||
size_t dlen = SINGLE_BLOCK_SIZE; // 65535
|
||
uint32_t crc = le_to_u32(addr + curReadPos + blockLen - 8);
|
||
int ret = bgzfUncompress(buf.data, &dlen, (Bytef*)(addr + curReadPos) + BLOCK_HEADER_LENGTH, blockLen - BLOCK_HEADER_LENGTH, crc);
|
||
if (ret != 0) {
|
||
spdlog::error("block len: {}, uncompressed len: {}", blockLen, dlen);
|
||
exit(0);
|
||
}
|
||
|
||
curReadPos += blockLen;
|
||
} else {
|
||
spdlog::error("not valid compressed block: {}, {}", curReadPos + blockLen, len);
|
||
break; /* 当前block数据不完整，一部分在还没读入的file数据里 */
|
||
}
|
||
}
|
||
if (curReadPos != len) {
|
||
spdlog::error("addr: {}, len: {}", curReadPos, len);
|
||
exit(0);
|
||
}
|
||
}
|
||
|
||
static void doWrite(Phase1PipelineArg& p) {
|
||
PROF_G_BEG(write_mid);
|
||
DataBuffer& compressBuf = p.compressBuf[p.writeOrder % p.COMPRESS_BUF_NUM];
|
||
// checkCompress(compressBuf.data, compressBuf.curLen);
|
||
fwrite(compressBuf.data, 1, compressBuf.curLen, p.midFilePtr);
|
||
PROF_G_END(write_mid);
|
||
}
|
||
|
||
void* phase1Write(void* data) {
|
||
Phase1PipelineArg& p = *(Phase1PipelineArg*)data;
|
||
// for test，写header
|
||
// fwrite(nsgv::gInHdr.compressed.data, 1, nsgv::gInHdr.compressed.curLen, p.midFilePtr);
|
||
|
||
/* do the work */
|
||
while (true) {
|
||
// previous dependency
|
||
yarn::DEPENDENCY_NOT_TO_BE(p.compressSig, 0);
|
||
|
||
if (p.compressFinish) {
|
||
while (p.writeOrder < p.compressOrder) {
|
||
doWrite(p);
|
||
p.writeOrder += 1;
|
||
}
|
||
break;
|
||
}
|
||
doWrite(p);
|
||
// update status
|
||
yarn::CONSUME_SIGNAL(p.compressSig);
|
||
p.writeOrder += 1;
|
||
}
|
||
|
||
// 写结尾，空block，bam文件需要
|
||
//fwrite("\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0", 1, 28, p.midFilePtr);
|
||
|
||
spdlog::info("End write order: {}", p.writeOrder);
|
||
return nullptr;
|
||
} |