diff --git a/example/MemMonitor.h b/example/MemMonitor.h index 2128928..ede26bf 100644 --- a/example/MemMonitor.h +++ b/example/MemMonitor.h @@ -5,6 +5,7 @@ #include #include #include +#include #include // 获取当前进程 RSS (物理内存) 使用量,单位 MB @@ -47,4 +48,130 @@ static inline void printMemUsage(const char *tag, const char *extra) printf("[MEM] %-30s | VmRSS: %6ld MB | " fmt "\n", tag, _rss / 1024, ##__VA_ARGS__); \ } while(0) +// 获取 GPU 显存使用情况(通过 nvidia-smi),返回所有 GPU 累加的已用/总量,单位 MB +// 成功返回 0,失败返回 -1(此时 usedMB/totalMB 置为 -1) +static inline int getGPUMemInfo(long &usedMB, long &totalMB) +{ + usedMB = -1; + totalMB = -1; + FILE *fp = popen("nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null", "r"); + if (!fp) + { + return -1; + } + char line[256]; + long used = 0; + long total = 0; + int n = 0; + while (fgets(line, sizeof(line), fp)) + { + long u = 0; + long t = 0; + if (sscanf(line, "%ld, %ld", &u, &t) == 2) + { + used += u; + total += t; + n++; + } + } + pclose(fp); + if (n == 0) + { + return -1; + } + usedMB = used; + totalMB = total; + return 0; +} + +// 获取系统内存信息(/proc/meminfo),单位 KB +// 成功返回 0,失败返回 -1(此时各值置为 -1) +static inline int getSystemMemInfo(long &totalKB, long &freeKB, long &availableKB) +{ + totalKB = -1; + freeKB = -1; + availableKB = -1; + FILE *fp = fopen("/proc/meminfo", "r"); + if (!fp) + { + return -1; + } + char line[256]; + while (fgets(line, sizeof(line), fp)) + { + long val = -1; + if (sscanf(line, "MemTotal: %ld kB", &val) == 1) + { + totalKB = val; + } + else if (sscanf(line, "MemFree: %ld kB", &val) == 1) + { + freeKB = val; + } + else if (sscanf(line, "MemAvailable: %ld kB", &val) == 1) + { + availableKB = val; + } + } + fclose(fp); + return 0; +} + +// 系统 CPU 时间快照(/proc/stat 第一行 cpu 累计值),用于两次采样计算 CPU 使用率 +struct CPUStat +{ + unsigned long long user; + unsigned long long nice; + unsigned long long system; + unsigned long long idle; + unsigned long long iowait; + unsigned long long irq; + unsigned long long softirq; + unsigned long long steal; + + CPUStat() + { + user = nice = system = idle = iowait = irq = softirq = steal = 0; + } + unsigned long long total() const + { + return user + nice + system + idle + iowait + irq + softirq + steal; + } + unsigned long long idleTotal() const + { + return idle + iowait; + } +}; + +// 读取系统 CPU 时间快照,成功返回 0,失败返回 -1 +static inline int getSystemCPUStat(CPUStat &stat) +{ + FILE *fp = fopen("/proc/stat", "r"); + if (!fp) + { + return -1; + } + char line[256]; + int ret = -1; + if (fgets(line, sizeof(line), fp)) + { + // 格式: cpu user nice system idle iowait irq softirq steal guest guest_nice + if (sscanf(line, "cpu %llu %llu %llu %llu %llu %llu %llu %llu", + &stat.user, &stat.nice, &stat.system, &stat.idle, + &stat.iowait, &stat.irq, &stat.softirq, &stat.steal) >= 4) + { + ret = 0; + } + } + fclose(fp); + return ret; +} + +// 进程累计消耗的 CPU 时间(用户态 + 内核态),单位秒 +static inline double cpuTimeSec(const struct rusage &usage) +{ + return (double)usage.ru_utime.tv_sec + (double)usage.ru_utime.tv_usec / 1e6 + + (double)usage.ru_stime.tv_sec + (double)usage.ru_stime.tv_usec / 1e6; +} + #endif // _MEM_MONITOR_H_ diff --git a/example/deal.cpp b/example/deal.cpp index 21b30ea..925f60a 100644 --- a/example/deal.cpp +++ b/example/deal.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "MemMonitor.h" #include #include @@ -1326,13 +1327,320 @@ int deal::DetImg() return 0; } +/// @brief 监控模式:每 5 分钟执行一次模型推理,并把当前设备资源消耗(CPU 内存 / GPU 显存) +/// 写入 monitor.log;当某次推理输出 out 图至少一个像素不为全黑时,保存该 out 图并退出监控。 +/// @return 0 表示成功,非 0 表示初始化失败 +int deal::MonitorRun() +{ + // 输入目录:优先使用命令行指定路径,否则使用默认路径 + std::string strSrcRoot = runConfig.filePath; + if (strSrcRoot.empty()) + { + strSrcRoot = "/home/aidlux/BOE/FOG/AI_Detect/"; + } + else if (strSrcRoot.back() != '/') + { + strSrcRoot += "/"; + } + + std::shared_ptr pAIFactory = AIFactory::GetInstance(); + std::shared_ptr pAI_Model = pAIFactory->AI_defect_NF; + if (!pAI_Model || !pAI_Model->m_bInitSuccess) + { + printf("MonitorRun: AI model (AI_defect_NF) not ready!\n"); + return 1; + } + + const int nModelInCh = pAI_Model->input_0.channel; + const int nModelInW = pAI_Model->input_0.width; + const int nModelInH = pAI_Model->input_0.height; + + printf("MonitorRun: model input [c=%d w=%d h=%d] output [c=%d w=%d h=%d]\n", + nModelInCh, nModelInW, nModelInH, + pAI_Model->output_0.channel, pAI_Model->output_0.width, pAI_Model->output_0.height); + + // 收集所有 *_in.png 文件 + std::vector inList; + try + { + if (!fs::exists(strSrcRoot)) + { + printf("MonitorRun: src dir not exist: %s\n", strSrcRoot.c_str()); + return 1; + } + for (const auto &entry : fs::recursive_directory_iterator(strSrcRoot)) + { + if (!entry.is_regular_file()) + { + continue; + } + std::string name = entry.path().filename().string(); + const std::string suffix = "_in.png"; + if (name.size() >= suffix.size() && + name.compare(name.size() - suffix.size(), suffix.size(), suffix) == 0) + { + inList.push_back(entry.path().string()); + } + } + } + catch (const std::exception &e) + { + printf("MonitorRun: traverse error %s\n", e.what()); + return 1; + } + + std::sort(inList.begin(), inList.end()); + printf("MonitorRun: total %zu in images\n", inList.size()); + if (inList.empty()) + { + printf("MonitorRun: no *_in.png found\n"); + return 1; + } + + // 预处理成模型输入(只读一次,每轮复用) + std::vector inImgs; + inImgs.reserve(inList.size()); + for (size_t i = 0; i < inList.size(); i++) + { + cv::Mat inImg = cv::imread(inList[i], cv::IMREAD_UNCHANGED); + if (inImg.empty()) + { + printf("MonitorRun: read fail: %s\n", inList[i].c_str()); + continue; + } + if (inImg.channels() != nModelInCh) + { + cv::Mat tmp; + if (inImg.channels() >= 3) + { + cv::cvtColor(inImg, tmp, cv::COLOR_BGR2GRAY); + } + else + { + tmp = inImg.clone(); + } + inImg = tmp; + } + if (inImg.type() != CV_8U) + { + inImg.convertTo(inImg, CV_8U); + } + if (inImg.cols != nModelInW || inImg.rows != nModelInH) + { + cv::resize(inImg, inImg, cv::Size(nModelInW, nModelInH)); + } + inImgs.push_back(inImg); + } + if (inImgs.empty()) + { + printf("MonitorRun: no valid in image\n"); + return 1; + } + + // 输出目录:输入目录去掉末尾 '/' 后追加 "_TEST/"(与 TestAIDet 保持一致) + std::string strDstRoot = strSrcRoot; + while (strDstRoot.size() > 1 && strDstRoot.back() == '/') + { + strDstRoot.pop_back(); + } + strDstRoot += "_TEST/"; + + // 本次监控的时间戳输出目录(存图和 log 都放这里) + time_t startT = time(NULL); + struct tm stmv; + localtime_r(&startT, &stmv); + char dirTs[64] = {0}; + strftime(dirTs, sizeof(dirTs), "monitor_%Y%m%d_%H%M%S", &stmv); + std::string strOutDir = strDstRoot + dirTs + "/"; + { + std::error_code ec; + fs::create_directories(strOutDir, ec); + } + + // 打开监控日志(追加写) + std::string strLogPath = strOutDir + "monitor.log"; + std::ofstream logFile(strLogPath, std::ios::app); + if (!logFile) + { + printf("MonitorRun: open log fail: %s\n", strLogPath.c_str()); + return 1; + } + + // 同时输出到终端和 log 文件 + auto logLine = [&logFile](const std::string &line) + { + printf("%s\n", line.c_str()); + logFile << line << std::endl; + logFile.flush(); + }; + + const int nIntervalSec = 5 * 60; // 5 分钟 + const int nMaxRunSec = 30 * 60; // 30 分钟 + long nRound = 0; + auto monitorStart = std::chrono::steady_clock::now(); + logLine("===== monitor start (run max 30 min) ====="); + + // 初始化 CPU 采样基准(用于计算 CPU 使用率) + struct rusage usagePrev; + getrusage(RUSAGE_SELF, &usagePrev); + CPUStat cpuStatPrev; + getSystemCPUStat(cpuStatPrev); + auto wallPrev = std::chrono::steady_clock::now(); + + while (true) + { + // 0. 运行 30 分钟后停止 + double elapsedSec = std::chrono::duration( + std::chrono::steady_clock::now() - monitorStart) + .count(); + if (elapsedSec >= nMaxRunSec) + { + logLine("===== monitor exit (timeout 30 min) ====="); + break; + } + + nRound++; + + // 1. 采集设备资源消耗 + long rssKB = getRSS_KB(); + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + + long memTotalKB = -1; + long memFreeKB = -1; + long memAvailKB = -1; + getSystemMemInfo(memTotalKB, memFreeKB, memAvailKB); + + CPUStat cpuStat; + getSystemCPUStat(cpuStat); + + long gpuUsedMB = -1; + long gpuTotalMB = -1; + getGPUMemInfo(gpuUsedMB, gpuTotalMB); + + auto wallNow = std::chrono::steady_clock::now(); + + // 系统 CPU 使用率(两次采样之间) + double sysCpuUsage = -1.0; + unsigned long long totalDelta = cpuStat.total() - cpuStatPrev.total(); + unsigned long long idleDelta = cpuStat.idleTotal() - cpuStatPrev.idleTotal(); + if (totalDelta > 0) + { + sysCpuUsage = 100.0 * (1.0 - (double)idleDelta / (double)totalDelta); + } + + // 进程 CPU 使用率(getrusage 的 CPU 时间 / 墙钟时间,多核下可能超过 100%) + double procCpuUsage = -1.0; + double wallSec = std::chrono::duration(wallNow - wallPrev).count(); + if (wallSec > 0) + { + procCpuUsage = 100.0 * (cpuTimeSec(usage) - cpuTimeSec(usagePrev)) / wallSec; + } + + // 更新采样基准 + usagePrev = usage; + cpuStatPrev = cpuStat; + wallPrev = wallNow; + + time_t now = time(NULL); + struct tm tmv; + localtime_r(&now, &tmv); + char ts[64] = {0}; + strftime(ts, sizeof(ts), "%Y-%m-%d %H:%M:%S", &tmv); + + char buf[512]; + snprintf(buf, sizeof(buf), + "[round %ld] %s | VmRSS: %ld MB | maxRSS: %ld MB | ProcCPU: %.1f%% | SysCPU: %.1f%% | Mem total %ld / free %ld / avail %ld MB | GPU: %ld/%ld MB", + nRound, ts, rssKB / 1024, usage.ru_maxrss / 1024, + procCpuUsage, sysCpuUsage, + memTotalKB / 1024, memFreeKB / 1024, memAvailKB / 1024, + gpuUsedMB, gpuTotalMB); + logLine(buf); + + // 2. 逐图推理,遇到 out 非全黑则保存对应的输入图和输出图 + int nHit = 0; + for (size_t i = 0; i < inImgs.size(); i++) + { + cv::Mat outImg; + int re = pAI_Model->AIDet(inImgs[i], outImg); + if (re != 0 || outImg.empty()) + { + snprintf(buf, sizeof(buf), + "[round %ld] infer fail(%d) on image %zu: %s", + nRound, re, i, inList[i].c_str()); + logLine(buf); + continue; + } + + int nNonZero = cv::countNonZero(outImg); + if (nNonZero > 0) + { + // 输出图不全黑 -> 保存对应的输入图和输出图 + std::string saveInPath = strOutDir + "monitor_hit_round" + + std::to_string(nRound) + "_img" + + std::to_string(i) + "_in.png"; + std::string saveOutPath = strOutDir + "monitor_hit_round" + + std::to_string(nRound) + "_img" + + std::to_string(i) + "_out.png"; + cv::imwrite(saveInPath, inImgs[i]); + cv::imwrite(saveOutPath, outImg); + + snprintf(buf, sizeof(buf), + "[round %ld] hit! out image %zu non-zero pixels: %d -> save %s / %s", + nRound, i, nNonZero, saveInPath.c_str(), saveOutPath.c_str()); + logLine(buf); + nHit++; + } + } + + snprintf(buf, sizeof(buf), "[round %ld] %d non-black out saved, sleep %d s ...", + nRound, nHit, nIntervalSec); + logLine(buf); + + // 3. 休眠(不超过剩余运行时间) + double remainSec = nMaxRunSec - elapsedSec; + int sleepSec = (int)(remainSec < nIntervalSec ? remainSec : nIntervalSec); + if (sleepSec > 0) + { + std::this_thread::sleep_for(std::chrono::seconds(sleepSec)); + } + } + + logFile.close(); + return 0; +} + /// @brief AI 检测独立测试:遍历 /home/aidlux/BOE/FOG/AI_Detect/ 下所有 *_in.png, /// 送入 AI_defect_NF 模型推理,将输出 *_out.png 保存到 /home/aidlux/BOE/FOG/AI_Detect_TEST/ /// @return 0 表示成功 int deal::TestAIDet() { - std::string strSrcRoot = "/home/aidlux/BOE/FOG/AI_Detect/"; - std::string strDstRoot = "/home/aidlux/BOE/FOG/AI_Detect_TEST/"; + // 输入目录:优先使用命令行指定路径,否则使用默认路径 + std::string strSrcRoot = runConfig.filePath; + if (strSrcRoot.empty()) + { + strSrcRoot = "/home/aidlux/BOE/FOG/AI_Detect/"; + } + else if (strSrcRoot.back() != '/') + { + strSrcRoot += "/"; + } + + // 输出目录:输入目录去掉末尾 '/' 后追加 "_TEST/" + // 默认输入 /home/aidlux/BOE/FOG/AI_Detect/ -> 输出 /home/aidlux/BOE/FOG/AI_Detect_TEST/ + std::string strDstRoot = strSrcRoot; + while (strDstRoot.size() > 1 && strDstRoot.back() == '/') + { + strDstRoot.pop_back(); + } + strDstRoot += "_TEST/"; + + if(runConfig.bmonitor) + { + // 监控模式:每 5 分钟执行一次模型推理,记录设备资源消耗到 log; + // 当某次推理输出 out 图至少一个像素不为全黑时退出监控。 + return MonitorRun(); + } std::shared_ptr pAIFactory = AIFactory::GetInstance(); std::shared_ptr pAI_Model = pAIFactory->AI_defect_NF; diff --git a/example/deal.h b/example/deal.h index c27d844..2f0ae07 100644 --- a/example/deal.h +++ b/example/deal.h @@ -548,6 +548,7 @@ struct DealRunConfig std::string filePath = ""; // 处理文件路径 如果为空 处理当前的单张图片 bool det_Tag = false; // 是否检测标签 bool bDebugSaveImg = false; // 是否中间图片图片 + bool bmonitor = false; // 是否监控 int m_nTestNum = 999999; void print(std::string str) { @@ -635,6 +636,7 @@ private: int ReJson_ALL(); int DetImg(); // 套图复测 int TestAIDet(); // AI 检测独立测试:读取 in 图送入模型,保存 out 图 + int MonitorRun(); // 监控模式:每 5 分钟推理一次,记录资源消耗,遇非全黑 out 退出 int ReJson_product(std::vector &jsonList, std::string strProductName); // 套图复测 int CheckFileImg(); ///< 检查文件图片(新增) int CheckProduct(const LoadProductImages &product); ///< 检查单个产品(新增) diff --git a/example/test_example.cpp b/example/test_example.cpp index d104735..a8f51a8 100644 --- a/example/test_example.cpp +++ b/example/test_example.cpp @@ -191,6 +191,10 @@ int main(int argc, char *argv[]) else if (string(argv[i]) == "-testAIDet") { test.runConfig.det_Type = RUNTYPE_RUN_File_AI_DetTest; + if (argc == 3) + { + test.runConfig.filePath = string(argv[i + 1]); + } } else if (string(argv[i]) == "-feqx" && i + 1 < argc) { @@ -242,6 +246,10 @@ int main(int argc, char *argv[]) { test.runConfig.bDebugSaveImg = true; } + if (string(argv[i]) == "-m") + { + test.runConfig.bmonitor = true; + } if (string(argv[i]) == "-n" && i + 1 < argc) { std::string strnum = string(argv[i + 1]);