/* Description: 第一阶段写入中间文件 Copyright : All right reserved by ICT Author : Zhang Zhonghai Date : 2026/06/02 */ #include "phase_1_write.h" #include #include #include #include #include #include "common_data.h" #include "const_val.h" #include "phase_1.h" #include "phase_1_compress.h" #include "phase_1_write.h" #include "sort.h" #include "util/profiling.h" static void checkCompress(uint8_t *addr, uint64_t len) { size_t curReadPos = 0; int blockLen = 0; int maxBlockLen = 0; DataBuffer buf; buf.AllocMem(SINGLE_BLOCK_SIZE); while (curReadPos + BLOCK_HEADER_LENGTH <= len) { /* 确保能解析block长度 */ blockLen = unpackInt16(&addr[curReadPos + 16]) + 1; if (blockLen > maxBlockLen) { maxBlockLen = blockLen; } if (curReadPos + blockLen <= len) { /* 完整的block数据在buf里 */ size_t dlen = SINGLE_BLOCK_SIZE; // 65535 uint32_t crc = le_to_u32(addr + curReadPos + blockLen - 8); int ret = bgzfUncompress(buf.data, &dlen, (Bytef*)(addr + curReadPos) + BLOCK_HEADER_LENGTH, blockLen - BLOCK_HEADER_LENGTH, crc); if (ret != 0) { spdlog::error("block len: {}, uncompressed len: {}", blockLen, dlen); exit(0); } curReadPos += blockLen; } else { spdlog::error("not valid compressed block: {}, {}", curReadPos + blockLen, len); break; /* 当前block数据不完整,一部分在还没读入的file数据里 */ } } if (curReadPos != len) { spdlog::error("addr: {}, len: {}", curReadPos, len); exit(0); } } static void doWrite(Phase1PipelineArg& p) { PROF_G_BEG(write_mid); DataBuffer& compressBuf = p.compressBuf[p.writeOrder % p.COMPRESS_BUF_NUM]; // checkCompress(compressBuf.data, compressBuf.curLen); fwrite(compressBuf.data, 1, compressBuf.curLen, p.midFilePtr); PROF_G_END(write_mid); } void* phase1Write(void* data) { Phase1PipelineArg& p = *(Phase1PipelineArg*)data; // for test,写header // fwrite(nsgv::gInHdr.compressed.data, 1, nsgv::gInHdr.compressed.curLen, p.midFilePtr); /* do the work */ while (true) { // previous dependency yarn::DEPENDENCY_NOT_TO_BE(p.compressSig, 0); if (p.compressFinish) { while (p.writeOrder < p.compressOrder) { doWrite(p); p.writeOrder += 1; } break; } doWrite(p); // update status yarn::CONSUME_SIGNAL(p.compressSig); p.writeOrder += 1; } // 写结尾,空block,bam文件需要 //fwrite("\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0", 1, 28, p.midFilePtr); spdlog::info("End write order: {}", p.writeOrder); return nullptr; }