171 lines
5.8 KiB
Python
171 lines
5.8 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
现代化日志配置系统 - 使用 loguru
|
||
解决多线程、中文编码、重复输出等问题
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
from loguru import logger
|
||
import threading
|
||
from pathlib import Path
|
||
|
||
class ModernLogger:
|
||
"""现代化日志管理器，基于loguru实现"""
|
||
|
||
def __init__(self):
|
||
self.initialized = False
|
||
self._lock = threading.Lock()
|
||
|
||
def setup_logging(self, log_dir=".", log_level="INFO", enable_console=True, enable_file=True):
|
||
"""
|
||
设置日志系统
|
||
|
||
参数:
|
||
log_dir: 日志目录
|
||
log_level: 日志级别
|
||
enable_console: 是否启用控制台输出
|
||
enable_file: 是否启用文件输出
|
||
"""
|
||
with self._lock:
|
||
if self.initialized:
|
||
return
|
||
|
||
# 清除默认的logger
|
||
logger.remove()
|
||
|
||
# 设置UTF-8编码环境
|
||
os.environ['PYTHONIOENCODING'] = 'utf-8'
|
||
os.environ['PYTHONLEGACYWINDOWSSTDIO'] = '0'
|
||
|
||
# Windows系统额外配置
|
||
if os.name == 'nt':
|
||
try:
|
||
os.system('chcp 65001 >nul 2>&1')
|
||
if hasattr(sys.stdout, 'reconfigure'):
|
||
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
||
sys.stderr.reconfigure(encoding='utf-8', errors='replace')
|
||
except Exception:
|
||
pass
|
||
|
||
# 控制台输出配置
|
||
if enable_console:
|
||
logger.add(
|
||
sys.stdout,
|
||
format="<green>{time:HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>线程{thread}</cyan> | <level>{message}</level>",
|
||
level=log_level,
|
||
colorize=True,
|
||
backtrace=True,
|
||
diagnose=True,
|
||
enqueue=True, # 多线程安全
|
||
catch=True
|
||
)
|
||
|
||
# 文件输出配置
|
||
if enable_file:
|
||
log_file = Path(log_dir) / "api_{time:YYYY-MM-DD}.log"
|
||
logger.add(
|
||
str(log_file),
|
||
format="{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | 线程{thread} | {message}",
|
||
level=log_level,
|
||
rotation="00:00", # 每天轮转
|
||
retention="7 days", # 保留7天
|
||
compression="zip", # 压缩旧日志
|
||
encoding="utf-8",
|
||
enqueue=True, # 多线程安全
|
||
catch=True
|
||
)
|
||
|
||
# 错误日志单独文件
|
||
error_log_file = Path(log_dir) / "api_error_{time:YYYY-MM-DD}.log"
|
||
logger.add(
|
||
str(error_log_file),
|
||
format="{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | 线程{thread} | {file}:{line} | {message}",
|
||
level="ERROR",
|
||
rotation="00:00",
|
||
retention="30 days", # 错误日志保留更久
|
||
compression="zip",
|
||
encoding="utf-8",
|
||
enqueue=True,
|
||
catch=True
|
||
)
|
||
|
||
self.initialized = True
|
||
logger.info("🚀 现代化日志系统初始化完成")
|
||
logger.info(f"📁 日志目录: {log_dir}")
|
||
logger.info(f"📊 日志级别: {log_level}")
|
||
logger.info(f"🖥️ 控制台输出: {'启用' if enable_console else '禁用'}")
|
||
logger.info(f"📄 文件输出: {'启用' if enable_file else '禁用'}")
|
||
|
||
def get_training_logger(self, task_id: str, model_type: str, product_id: str):
|
||
"""
|
||
获取训练专用的logger
|
||
|
||
参数:
|
||
task_id: 训练任务ID
|
||
model_type: 模型类型
|
||
product_id: 产品ID
|
||
|
||
返回:
|
||
配置好的logger实例
|
||
"""
|
||
return logger.bind(
|
||
task_id=task_id[:8],
|
||
model_type=model_type,
|
||
product_id=product_id,
|
||
context="TRAINING"
|
||
)
|
||
|
||
def get_api_logger(self):
|
||
"""获取API专用的logger"""
|
||
return logger.bind(context="API")
|
||
|
||
def log_training_progress(self, task_id: str, message: str, progress: float = None, **kwargs):
|
||
"""
|
||
记录训练进度日志
|
||
|
||
参数:
|
||
task_id: 任务ID
|
||
message: 日志消息
|
||
progress: 进度百分比
|
||
**kwargs: 额外的日志字段
|
||
"""
|
||
extra_info = ""
|
||
if progress is not None:
|
||
extra_info += f" [进度: {progress:.1f}%]"
|
||
|
||
for key, value in kwargs.items():
|
||
extra_info += f" [{key}: {value}]"
|
||
|
||
logger.bind(
|
||
task_id=task_id[:8],
|
||
context="TRAINING_PROGRESS"
|
||
).info(f"🔥 {message}{extra_info}")
|
||
|
||
# 全局logger实例
|
||
modern_logger = ModernLogger()
|
||
|
||
def get_logger():
|
||
"""获取配置好的logger实例"""
|
||
if not modern_logger.initialized:
|
||
modern_logger.setup_logging()
|
||
return logger
|
||
|
||
def setup_api_logging(log_dir=".", log_level="INFO"):
|
||
"""API服务器日志初始化的便捷函数"""
|
||
modern_logger.setup_logging(
|
||
log_dir=log_dir,
|
||
log_level=log_level,
|
||
enable_console=True,
|
||
enable_file=True
|
||
)
|
||
return get_logger()
|
||
|
||
def get_training_logger(task_id: str, model_type: str, product_id: str):
|
||
"""获取训练专用logger的便捷函数"""
|
||
return modern_logger.get_training_logger(task_id, model_type, product_id)
|
||
|
||
def log_training_progress(task_id: str, message: str, **kwargs):
|
||
"""记录训练进度的便捷函数"""
|
||
modern_logger.log_training_progress(task_id, message, **kwargs) |