|
|
|
|
@ -1327,9 +1327,9 @@ int deal::DetImg()
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// @brief 监控模式:每 10 秒执行一次模型推理,并把当前设备资源消耗(CPU 内存 / GPU 显存)
|
|
|
|
|
/// 写入 monitor.log;out 图重复覆盖保存到输入图同名 out 文件(*_in.png -> *_out.png),
|
|
|
|
|
/// 当某次推理输出 out 图至少一个像素不为全黑时退出监控。
|
|
|
|
|
/// @brief 监控模式:4 线程并发循环执行模型推理(压 GPU),并把当前设备资源消耗
|
|
|
|
|
/// (CPU 内存 / GPU 显存)写入 monitor.log;正常(out 全黑)不写图,检测到残点
|
|
|
|
|
/// (out 图至少一个像素不为全黑)时才保存对应 *_out.png 并退出监控。
|
|
|
|
|
/// @return 0 表示成功,非 0 表示初始化失败
|
|
|
|
|
int deal::MonitorRun()
|
|
|
|
|
{
|
|
|
|
|
@ -1474,26 +1474,96 @@ int deal::MonitorRun()
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 同时输出到终端和 log 文件
|
|
|
|
|
auto logLine = [&logFile](const std::string &line)
|
|
|
|
|
// 日志互斥锁(worker 线程与主线程都会写日志)
|
|
|
|
|
std::mutex logMutex;
|
|
|
|
|
auto logLine = [&logFile, &logMutex](const std::string &line)
|
|
|
|
|
{
|
|
|
|
|
std::lock_guard<std::mutex> lock(logMutex);
|
|
|
|
|
printf("%s\n", line.c_str());
|
|
|
|
|
fflush(stdout); // 强制刷新控制台,避免 stdout 块缓冲导致显示延迟
|
|
|
|
|
logFile << line << std::endl;
|
|
|
|
|
logFile.flush();
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const int nIntervalSec = 1; // 1 秒
|
|
|
|
|
long nRound = 0;
|
|
|
|
|
logLine("===== monitor start (every 10s, exit on first non-black out) =====");
|
|
|
|
|
const int nThreadNum = 4; // 并发线程数(与模型 Stream_num 一致时并行度最高)
|
|
|
|
|
long nImageNum = (long)inImgs.size();
|
|
|
|
|
|
|
|
|
|
// 共享状态
|
|
|
|
|
std::atomic<long> nNextIdx{0}; // 原子分配任务索引,轮询所有图
|
|
|
|
|
std::atomic<long> nInferCount{0}; // 成功推理次数
|
|
|
|
|
std::atomic<long> nFailCount{0}; // 推理失败次数
|
|
|
|
|
std::atomic<bool> bHit{false}; // 是否命中非全黑 out
|
|
|
|
|
std::atomic<long> nHitIdx{-1}; // 命中的图索引
|
|
|
|
|
std::atomic<int> nHitNonZero{0}; // 命中的非零像素数
|
|
|
|
|
|
|
|
|
|
std::mutex saveMutex; // 保护 out 文件写入
|
|
|
|
|
|
|
|
|
|
logLine("===== monitor start (4 threads loop, exit on first non-black out) =====");
|
|
|
|
|
|
|
|
|
|
// 启动 worker 线程:循环推理压 GPU
|
|
|
|
|
std::vector<std::thread> workers;
|
|
|
|
|
workers.reserve(nThreadNum);
|
|
|
|
|
for (int t = 0; t < nThreadNum; t++)
|
|
|
|
|
{
|
|
|
|
|
workers.emplace_back([&, t]()
|
|
|
|
|
{
|
|
|
|
|
while (!bHit.load(std::memory_order_relaxed))
|
|
|
|
|
{
|
|
|
|
|
long idx = nNextIdx.fetch_add(1, std::memory_order_relaxed) % nImageNum;
|
|
|
|
|
|
|
|
|
|
cv::Mat outImg;
|
|
|
|
|
int re = pAI_Model->AIDet(inImgs[(size_t)idx], outImg);
|
|
|
|
|
if (re != 0 || outImg.empty())
|
|
|
|
|
{
|
|
|
|
|
nFailCount.fetch_add(1, std::memory_order_relaxed);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 初始化 CPU 采样基准(用于计算 CPU 使用率)
|
|
|
|
|
nInferCount.fetch_add(1, std::memory_order_relaxed);
|
|
|
|
|
|
|
|
|
|
int nNonZero = cv::countNonZero(outImg);
|
|
|
|
|
if (nNonZero > 0)
|
|
|
|
|
{
|
|
|
|
|
// 有残点才写图,然后退出
|
|
|
|
|
bool expected = false;
|
|
|
|
|
if (bHit.compare_exchange_strong(expected, true))
|
|
|
|
|
{
|
|
|
|
|
{
|
|
|
|
|
std::lock_guard<std::mutex> lock(saveMutex);
|
|
|
|
|
fs::path outParent = fs::path(outList[(size_t)idx]).parent_path();
|
|
|
|
|
if (!outParent.empty())
|
|
|
|
|
{
|
|
|
|
|
std::error_code ec;
|
|
|
|
|
fs::create_directories(outParent, ec);
|
|
|
|
|
}
|
|
|
|
|
cv::imwrite(outList[(size_t)idx], outImg);
|
|
|
|
|
}
|
|
|
|
|
nHitIdx.store(idx, std::memory_order_relaxed);
|
|
|
|
|
nHitNonZero.store(nNonZero, std::memory_order_relaxed);
|
|
|
|
|
logLine("[hit] thread " + std::to_string(t) + " out image " +
|
|
|
|
|
okInList[(size_t)idx] + " non-zero pixels: " +
|
|
|
|
|
std::to_string(nNonZero) + " -> " + outList[(size_t)idx]);
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 主线程:资源监控(持续循环采样),直到命中退出
|
|
|
|
|
struct rusage usagePrev;
|
|
|
|
|
getrusage(RUSAGE_SELF, &usagePrev);
|
|
|
|
|
CPUStat cpuStatPrev;
|
|
|
|
|
getSystemCPUStat(cpuStatPrev);
|
|
|
|
|
auto wallPrev = std::chrono::steady_clock::now();
|
|
|
|
|
|
|
|
|
|
while (true)
|
|
|
|
|
long nRound = 0;
|
|
|
|
|
|
|
|
|
|
// GPU 显存采样很慢(内部执行 nvidia-smi),降低采样频率避免阻塞主循环
|
|
|
|
|
long gpuUsedMB = -1;
|
|
|
|
|
long gpuTotalMB = -1;
|
|
|
|
|
const int nGPUSampleInterval = 10; // 每 10 轮(约 10 秒)才调用一次 nvidia-smi
|
|
|
|
|
|
|
|
|
|
while (!bHit.load(std::memory_order_relaxed))
|
|
|
|
|
{
|
|
|
|
|
nRound++;
|
|
|
|
|
|
|
|
|
|
@ -1510,9 +1580,10 @@ int deal::MonitorRun()
|
|
|
|
|
CPUStat cpuStat;
|
|
|
|
|
getSystemCPUStat(cpuStat);
|
|
|
|
|
|
|
|
|
|
long gpuUsedMB = -1;
|
|
|
|
|
long gpuTotalMB = -1;
|
|
|
|
|
if ((nRound % nGPUSampleInterval) == 1)
|
|
|
|
|
{
|
|
|
|
|
getGPUMemInfo(gpuUsedMB, gpuTotalMB);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
auto wallNow = std::chrono::steady_clock::now();
|
|
|
|
|
|
|
|
|
|
@ -1546,63 +1617,33 @@ int deal::MonitorRun()
|
|
|
|
|
|
|
|
|
|
char buf[512];
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
|
"[round %ld] %s | VmRSS: %ld MB | maxRSS: %ld MB | ProcCPU: %.1f%% | SysCPU: %.1f%% | Mem total %ld / free %ld / avail %ld MB | GPU: %ld/%ld MB",
|
|
|
|
|
nRound, ts, rssKB / 1024, usage.ru_maxrss / 1024,
|
|
|
|
|
"[round %ld] %s | infer=%ld fail=%ld | VmRSS: %ld MB | maxRSS: %ld MB | ProcCPU: %.1f%% | SysCPU: %.1f%% | Mem total %ld / free %ld / avail %ld MB | GPU: %ld/%ld MB",
|
|
|
|
|
nRound, ts, nInferCount.load(), nFailCount.load(),
|
|
|
|
|
rssKB / 1024, usage.ru_maxrss / 1024,
|
|
|
|
|
procCpuUsage, sysCpuUsage,
|
|
|
|
|
memTotalKB / 1024, memFreeKB / 1024, memAvailKB / 1024,
|
|
|
|
|
gpuUsedMB, gpuTotalMB);
|
|
|
|
|
logLine(buf);
|
|
|
|
|
|
|
|
|
|
// 2. 逐图推理,out 图重复覆盖保存;遇到非全黑则退出
|
|
|
|
|
bool bHit = false;
|
|
|
|
|
for (size_t i = 0; i < inImgs.size(); i++)
|
|
|
|
|
// 打印降频:每 10 轮打印一次,减少日志 IO/系统调用对 CPU 的争抢
|
|
|
|
|
if ((nRound % 10) == 1)
|
|
|
|
|
{
|
|
|
|
|
cv::Mat outImg;
|
|
|
|
|
int re = pAI_Model->AIDet(inImgs[i], outImg);
|
|
|
|
|
if (re != 0 || outImg.empty())
|
|
|
|
|
{
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
|
"[round %ld] infer fail(%d) on image %zu: %s",
|
|
|
|
|
nRound, re, i, okInList[i].c_str());
|
|
|
|
|
logLine(buf);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 输出图重复覆盖保存到 *_out.png
|
|
|
|
|
bool bSaved = false;
|
|
|
|
|
{
|
|
|
|
|
fs::path outParent = fs::path(outList[i]).parent_path();
|
|
|
|
|
if (!outParent.empty())
|
|
|
|
|
{
|
|
|
|
|
std::error_code ec;
|
|
|
|
|
fs::create_directories(outParent, ec);
|
|
|
|
|
}
|
|
|
|
|
bSaved = cv::imwrite(outList[i], outImg);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int nNonZero = cv::countNonZero(outImg);
|
|
|
|
|
if (nNonZero > 0)
|
|
|
|
|
// 等待所有 worker 线程退出
|
|
|
|
|
for (auto &th : workers)
|
|
|
|
|
{
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
|
"[round %ld] hit! out image %zu non-zero pixels: %d -> %s (saved=%d), exit monitor",
|
|
|
|
|
nRound, i, nNonZero, outList[i].c_str(), bSaved ? 1 : 0);
|
|
|
|
|
logLine(buf);
|
|
|
|
|
bHit = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (bHit)
|
|
|
|
|
if (th.joinable())
|
|
|
|
|
{
|
|
|
|
|
break;
|
|
|
|
|
th.join();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf), "[round %ld] all out black, sleep %d s ...", nRound, nIntervalSec);
|
|
|
|
|
logLine(buf);
|
|
|
|
|
|
|
|
|
|
std::this_thread::sleep_for(std::chrono::seconds(nIntervalSec));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
long hitIdx = nHitIdx.load();
|
|
|
|
|
logLine("===== monitor exit (hit image " +
|
|
|
|
|
(hitIdx >= 0 ? okInList[(size_t)hitIdx] : std::string("?")) +
|
|
|
|
|
", non-zero pixels " + std::to_string(nHitNonZero.load()) + ") =====");
|
|
|
|
|
|
|
|
|
|
logFile.close();
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|