解決了一个很诡异的bug，ThreadParam可能是由于命名的问题，导致vector内存错误
This commit is contained in:
parent
0c73318fb7
commit
215e1d3dea
|
|
@ -12,13 +12,16 @@
|
|||
#include <string>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
#include <filesystem>
|
||||
#include <mat.h>
|
||||
#include "common.h"
|
||||
#include "CommonLib/thread_pool.h"
|
||||
#include "CommonLib/matlab_io.h"
|
||||
#include "CommonLib/kthread.h"
|
||||
namespace fs = std::filesystem;
|
||||
using std::cout;
|
||||
using std::vector;
|
||||
using namespace std;
|
||||
|
||||
/* 将结果写入mat文件 */
|
||||
/* 将数据写入mat文件中，用给定的名称命名 */
|
||||
bool SavePubmed(const string& matPath,
|
||||
|
|
@ -71,7 +74,7 @@ bool SavePubmed(const string& matPath,
|
|||
return true;
|
||||
}
|
||||
/* 处理一篇文章 */
|
||||
struct ThreadParam { // 线程参数
|
||||
struct ThreadParamPubmed { // 线程参数
|
||||
unordered_map<string, string> *pumTagContent;
|
||||
vector<string> *pvLineTag;
|
||||
vector<string> *pvTgName;
|
||||
|
|
@ -81,9 +84,7 @@ struct ThreadParam { // 线程参数
|
|||
vector<string> *pvStrPubmedTxt;
|
||||
};
|
||||
|
||||
//void ThreadProcessArticle(vector<ThreadParam>& vTP, long idx, int tid) {
|
||||
void ThreadProcessArticle(ThreadParam& param) {
|
||||
//ThreadParam& param = vTP[idx];
|
||||
void ThreadProcessArticle(ThreadParamPubmed& param) {
|
||||
unordered_map<string, string>& umTagContent = *param.pumTagContent;
|
||||
vector<string>& vLineTag = *param.pvLineTag;
|
||||
vector<string>& vTgName = *param.pvTgName;
|
||||
|
|
@ -108,7 +109,7 @@ void ThreadProcessArticle(ThreadParam& param) {
|
|||
}
|
||||
|
||||
// 命令行参数示例
|
||||
// ProcessPubmedTxt d:\Twirls\gat1\literatures\pubmed_tag.mat D:\Twirls\runtime\negatives\pubmed-multiplesc-set.txt d:\pubmed_txt.mat 12
|
||||
// ProcessPubmedTxt d:\Twirls\gat1\literatures\pubmed_tag.mat D:\Twirls\runtime\pubmed_files d:\pubmed_txt.mat 12
|
||||
/*
|
||||
pubmed txt文件中包含多个文章的摘要信息，每个信息最前边有一个tag，每个tag对应的信息可能有一行，也可能多行，每个文章中间由一个空行隔开
|
||||
1. 读取预先提取的pubmed tags, 并将tags中的'-'和' '字符去掉，只留下纯字符串做tag
|
||||
|
|
@ -117,10 +118,10 @@ void ThreadProcessArticle(ThreadParam& param) {
|
|||
4. 将结果写入mat文件
|
||||
*/
|
||||
void ProcessPubmedTxt(int argc, const char** argv) {
|
||||
// argv 1.pubmed tag.mat文件; 2.pubmed article.txt文件; 3.pubmed out.mat输出文件
|
||||
// argv 1.pubmed tag.mat文件; 2.pubmed txt文件父目录; 3.pubmed out.mat输出文件; 4.Thread number
|
||||
//
|
||||
if (argc < 4) {
|
||||
cout << "This program should take at least 3 arguments(1.pubmed tag.mat; 2. pubmed article.txt; 3. pubmed out.mat; [4. thread num])!" << endl;
|
||||
cout << "This program should take at least 3 arguments(1.pubmed tag.mat; 2. pubmed txt parent dir; 3. pubmed out.mat; [4. thread num])!" << endl;
|
||||
return;
|
||||
}
|
||||
clock_t begin, finish;
|
||||
|
|
@ -128,7 +129,7 @@ void ProcessPubmedTxt(int argc, const char** argv) {
|
|||
vector<string> vTg;
|
||||
vector<string> vTgName;
|
||||
vector<unordered_map<string, string> > vumPaperTagVal;
|
||||
unordered_map<string, string> umFullTagToTag; // 完整tag与tag的映射，如“PMID- ”：“PMID”
|
||||
unordered_map<string, string> umFullTagToTag; // 完整tag与tag的映射，如“PMID- ”:“PMID”
|
||||
/* 读取pubmed tags */
|
||||
ReadMtxString(argv[1], "tg", vTg, &rowNum, &colNum);
|
||||
/* 1. 去掉tags里的'-'和' '字符，得到纯净的tag */
|
||||
|
|
@ -148,7 +149,8 @@ void ProcessPubmedTxt(int argc, const char** argv) {
|
|||
cout << "process tag Total time: " << (double)(finish - begin) / CLOCKS_PER_SEC << " s" << endl;
|
||||
|
||||
/* 2. 读取pubmed txt文件，先读入后处理 */
|
||||
ifstream ifsPubmedTxt(argv[2]);
|
||||
string parentDir(argv[2]);
|
||||
string txtSuffix(".txt");
|
||||
vector<string> vStrPubmedTxt;
|
||||
vector<string> vLineTag;
|
||||
vector<int> vPaperStartIdx;
|
||||
|
|
@ -159,6 +161,11 @@ void ProcessPubmedTxt(int argc, const char** argv) {
|
|||
vPaperStartIdx.push_back(curPos); // 添加初始索引
|
||||
const int FULL_TAG_LEN = 5;
|
||||
begin = clock();
|
||||
for (auto &file : fs::directory_iterator(parentDir)) { // 遍历目录里的每一个txt文件
|
||||
const string &fileName = file.path().filename().string();
|
||||
auto rPos = fileName.rfind(txtSuffix);
|
||||
if (rPos != string::npos && fileName.size() - rPos == txtSuffix.size()){
|
||||
ifstream ifsPubmedTxt(file.path().string());
|
||||
while (getline(ifsPubmedTxt, strLine)) { // 读取内容时候去掉了行尾的换行符
|
||||
while (strLine.back() == ' ') strLine.pop_back(); // 去掉行尾的空格
|
||||
if (strLine.size() == 0) { // 新的paper
|
||||
|
|
@ -177,21 +184,29 @@ void ProcessPubmedTxt(int argc, const char** argv) {
|
|||
}
|
||||
}
|
||||
vPaperStartIdx.push_back(curPos); // 比文章多1，最后一个记录结束位置
|
||||
ifsPubmedTxt.close();
|
||||
}
|
||||
}
|
||||
|
||||
finish = clock();
|
||||
cout << "read txt Total time: " << (double)(finish - begin) / CLOCKS_PER_SEC << " s" << endl;
|
||||
|
||||
cout << "paper num: " << vPaperStartIdx.size() - 1 << endl;
|
||||
/* 处理每一篇文章 */
|
||||
int numThread = 1;
|
||||
if (argc >= 5) numThread = atoi(argv[4]);
|
||||
if (numThread < 1) numThread = 1;
|
||||
ThreadPool thPool(numThread);
|
||||
// ThreadPool thPool(numThread);
|
||||
vumPaperTagVal.resize(vPaperStartIdx.size()-1);
|
||||
vector<thread> vT;
|
||||
vector<ThreadParam> vTP(vPaperStartIdx.size() - 1);
|
||||
vector<ThreadParamPubmed> vTP(vumPaperTagVal.size());
|
||||
begin = clock();
|
||||
for (int i = 0; i < vTP.size(); ++i) {
|
||||
for (int i = 0; i < vumPaperTagVal.size(); ++i) {
|
||||
vTP[i] = { &vumPaperTagVal[i], &vLineTag, &vTgName, vPaperStartIdx[i], vPaperStartIdx[i + 1], &umFullTagToTag, &vStrPubmedTxt };
|
||||
// ThreadParamPubmed tp = { &vumPaperTagVal[i], &vLineTag, &vTgName, vPaperStartIdx[i], vPaperStartIdx[i + 1], &umFullTagToTag, &vStrPubmedTxt };
|
||||
// ThreadProcessArticle(tp);
|
||||
// thPool.enqueue(ThreadProcessArticle, tp);
|
||||
}
|
||||
// thPool.~ThreadPool();
|
||||
|
||||
kt_for(numThread, ThreadProcessArticle, vTP);
|
||||
finish = clock();
|
||||
cout << "kt for Total time: " << (double)(finish - begin) / CLOCKS_PER_SEC << " s" << endl;
|
||||
|
|
@ -237,9 +252,6 @@ void ProcessPubmedTxt(int argc, const char** argv) {
|
|||
finish = clock();
|
||||
cout << "merge abs and title Total time : " << (double)(finish - begin) / CLOCKS_PER_SEC << " s" << endl;
|
||||
|
||||
// 关闭txt文件
|
||||
ifsPubmedTxt.close();
|
||||
|
||||
/* 将处理后的数据写入mat文件，mat中的变量名称分别为Tx和abs1 */
|
||||
begin = clock();
|
||||
SavePubmed(argv[3], vTgName, vumPaperTagVal);
|
||||
|
|
|
|||
Loading…
Reference in New Issue