750 lines
18 KiB
Rust
750 lines
18 KiB
Rust
//! 生产环境日志模块
|
|
//!
|
|
//! 提供生产环境专用的日志功能，包括结构化日志、日志聚合、性能监控等。
|
|
#![allow(dead_code)]
|
|
|
|
use anyhow::{Context, Result};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, SystemTime};
|
|
use tokio::sync::RwLock;
|
|
use tracing::error;
|
|
|
|
/// 生产日志管理器
|
|
#[derive(Clone)]
|
|
pub struct ProductionLogger {
|
|
/// 日志配置
|
|
config: LoggingConfig,
|
|
/// 日志收集器
|
|
collectors: Vec<Arc<dyn LogCollector + Send + Sync>>,
|
|
/// 日志过滤器
|
|
filters: Vec<Arc<dyn LogFilter + Send + Sync>>,
|
|
/// 日志统计
|
|
stats: Arc<RwLock<LoggingStats>>,
|
|
/// 日志缓冲区
|
|
buffer: Arc<RwLock<LogBuffer>>,
|
|
}
|
|
|
|
impl std::fmt::Debug for ProductionLogger {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("ProductionLogger")
|
|
.field("config", &self.config)
|
|
.field("collectors_count", &self.collectors.len())
|
|
.field("filters_count", &self.filters.len())
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
/// 日志配置
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LoggingConfig {
|
|
/// 日志级别
|
|
pub level: LogLevel,
|
|
/// 输出格式
|
|
pub format: LogFormat,
|
|
/// 输出目标
|
|
pub targets: Vec<LogTarget>,
|
|
/// 缓冲配置
|
|
pub buffer_config: BufferConfig,
|
|
/// 轮转配置
|
|
pub rotation_config: RotationConfig,
|
|
/// 采样配置
|
|
pub sampling_config: SamplingConfig,
|
|
}
|
|
|
|
/// 日志级别
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum LogLevel {
|
|
Trace,
|
|
Debug,
|
|
Info,
|
|
Warn,
|
|
Error,
|
|
}
|
|
|
|
/// 日志格式
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum LogFormat {
|
|
/// JSON 格式
|
|
Json,
|
|
/// 纯文本格式
|
|
Text,
|
|
/// 结构化格式
|
|
Structured,
|
|
/// 自定义格式
|
|
Custom(String),
|
|
}
|
|
|
|
/// 日志输出目标
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum LogTarget {
|
|
/// 控制台输出
|
|
Console,
|
|
/// 文件输出
|
|
File { path: String },
|
|
/// 系统日志
|
|
Syslog,
|
|
/// 远程日志服务
|
|
Remote { endpoint: String, api_key: String },
|
|
/// Elasticsearch
|
|
Elasticsearch { url: String, index: String },
|
|
/// 自定义目标
|
|
Custom(String),
|
|
}
|
|
|
|
/// 缓冲配置
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BufferConfig {
|
|
/// 缓冲区大小
|
|
pub size: usize,
|
|
/// 刷新间隔
|
|
pub flush_interval: Duration,
|
|
/// 批量大小
|
|
pub batch_size: usize,
|
|
/// 是否启用压缩
|
|
pub enable_compression: bool,
|
|
}
|
|
|
|
/// 轮转配置
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct RotationConfig {
|
|
/// 最大文件大小 (MB)
|
|
pub max_file_size_mb: u64,
|
|
/// 最大文件数量
|
|
pub max_files: u32,
|
|
/// 轮转间隔
|
|
pub rotation_interval: Duration,
|
|
/// 压缩旧文件
|
|
pub compress_old_files: bool,
|
|
}
|
|
|
|
/// 采样配置
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SamplingConfig {
|
|
/// 是否启用采样
|
|
pub enabled: bool,
|
|
/// 采样率 (0.0-1.0)
|
|
pub rate: f64,
|
|
/// 采样策略
|
|
pub strategy: SamplingStrategy,
|
|
}
|
|
|
|
/// 采样策略
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum SamplingStrategy {
|
|
/// 随机采样
|
|
Random,
|
|
/// 基于时间的采样
|
|
TimeBased,
|
|
/// 基于级别的采样
|
|
LevelBased,
|
|
/// 自适应采样
|
|
Adaptive,
|
|
}
|
|
|
|
/// 日志条目
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LogEntry {
|
|
/// 时间戳
|
|
pub timestamp: SystemTime,
|
|
/// 日志级别
|
|
pub level: LogLevel,
|
|
/// 消息
|
|
pub message: String,
|
|
/// 模块
|
|
pub module: String,
|
|
/// 文件名
|
|
pub file: Option<String>,
|
|
/// 行号
|
|
pub line: Option<u32>,
|
|
/// 字段
|
|
pub fields: HashMap<String, serde_json::Value>,
|
|
/// 跟踪 ID
|
|
pub trace_id: Option<String>,
|
|
/// 跨度 ID
|
|
pub span_id: Option<String>,
|
|
}
|
|
|
|
/// 日志收集器 trait
|
|
pub trait LogCollector {
|
|
/// 收集日志
|
|
fn collect(&self, entry: &LogEntry) -> Result<()>;
|
|
/// 刷新缓冲区
|
|
fn flush(&self) -> Result<()>;
|
|
/// 获取收集器名称
|
|
fn name(&self) -> &str;
|
|
}
|
|
|
|
/// 日志过滤器 trait
|
|
pub trait LogFilter {
|
|
/// 过滤日志
|
|
fn filter(&self, entry: &LogEntry) -> bool;
|
|
/// 获取过滤器名称
|
|
fn name(&self) -> &str;
|
|
}
|
|
|
|
/// 控制台日志收集器
|
|
pub struct ConsoleCollector {
|
|
/// 收集器名称
|
|
name: String,
|
|
/// 格式化器
|
|
formatter: Box<dyn LogFormatter + Send + Sync>,
|
|
}
|
|
|
|
impl std::fmt::Debug for ConsoleCollector {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("ConsoleCollector")
|
|
.field("name", &self.name)
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
/// 文件日志收集器
|
|
pub struct FileCollector {
|
|
/// 收集器名称
|
|
name: String,
|
|
/// 文件路径
|
|
file_path: String,
|
|
/// 轮转配置
|
|
rotation_config: RotationConfig,
|
|
/// 格式化器
|
|
formatter: Box<dyn LogFormatter + Send + Sync>,
|
|
}
|
|
|
|
impl std::fmt::Debug for FileCollector {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("FileCollector")
|
|
.field("name", &self.name)
|
|
.field("file_path", &self.file_path)
|
|
.field("rotation_config", &self.rotation_config)
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
/// 远程日志收集器
|
|
pub struct RemoteCollector {
|
|
/// 收集器名称
|
|
name: String,
|
|
/// 远程端点
|
|
endpoint: String,
|
|
/// API 密钥
|
|
api_key: String,
|
|
/// HTTP 客户端
|
|
client: reqwest::Client,
|
|
}
|
|
|
|
impl std::fmt::Debug for RemoteCollector {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("RemoteCollector")
|
|
.field("name", &self.name)
|
|
.field("endpoint", &self.endpoint)
|
|
.field("api_key", &"<hidden>")
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
/// 日志格式化器 trait
|
|
pub trait LogFormatter {
|
|
/// 格式化日志条目
|
|
fn format(&self, entry: &LogEntry) -> Result<String>;
|
|
}
|
|
|
|
/// JSON 格式化器
|
|
#[derive(Debug)]
|
|
pub struct JsonFormatter;
|
|
|
|
/// 文本格式化器
|
|
#[derive(Debug)]
|
|
pub struct TextFormatter {
|
|
/// 时间格式
|
|
time_format: String,
|
|
/// 是否包含颜色
|
|
colored: bool,
|
|
}
|
|
|
|
/// 级别过滤器
|
|
#[derive(Debug)]
|
|
pub struct LevelFilter {
|
|
/// 最小级别
|
|
min_level: LogLevel,
|
|
}
|
|
|
|
/// 模块过滤器
|
|
#[derive(Debug)]
|
|
pub struct ModuleFilter {
|
|
/// 允许的模块
|
|
allowed_modules: Vec<String>,
|
|
/// 禁止的模块
|
|
denied_modules: Vec<String>,
|
|
}
|
|
|
|
/// 速率限制过滤器
|
|
#[derive(Debug)]
|
|
pub struct RateLimitFilter {
|
|
/// 速率限制器
|
|
limiter: Arc<RwLock<HashMap<String, RateLimiter>>>,
|
|
/// 每秒最大日志数
|
|
max_logs_per_second: u32,
|
|
}
|
|
|
|
/// 简单速率限制器
|
|
#[derive(Debug)]
|
|
struct RateLimiter {
|
|
/// 最后重置时间
|
|
last_reset: SystemTime,
|
|
/// 当前计数
|
|
count: u32,
|
|
/// 限制
|
|
limit: u32,
|
|
}
|
|
|
|
/// 日志缓冲区
|
|
#[derive(Debug)]
|
|
pub struct LogBuffer {
|
|
/// 缓冲的日志条目
|
|
entries: Vec<LogEntry>,
|
|
/// 缓冲区大小限制
|
|
max_size: usize,
|
|
/// 最后刷新时间
|
|
last_flush: SystemTime,
|
|
}
|
|
|
|
/// 日志统计
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LoggingStats {
|
|
/// 总日志数
|
|
pub total_logs: u64,
|
|
/// 按级别统计
|
|
pub logs_by_level: HashMap<String, u64>,
|
|
/// 按模块统计
|
|
pub logs_by_module: HashMap<String, u64>,
|
|
/// 错误统计
|
|
pub error_count: u64,
|
|
/// 警告统计
|
|
pub warning_count: u64,
|
|
/// 平均处理时间
|
|
pub avg_processing_time: Duration,
|
|
/// 缓冲区使用率
|
|
pub buffer_usage: f64,
|
|
/// 丢弃的日志数
|
|
pub dropped_logs: u64,
|
|
}
|
|
|
|
impl ProductionLogger {
|
|
/// 创建新的生产日志管理器
|
|
pub fn new(config: LoggingConfig) -> Self {
|
|
Self {
|
|
config,
|
|
collectors: Vec::new(),
|
|
filters: Vec::new(),
|
|
stats: Arc::new(RwLock::new(LoggingStats::default())),
|
|
buffer: Arc::new(RwLock::new(LogBuffer::new(1000))),
|
|
}
|
|
}
|
|
|
|
/// 添加日志收集器
|
|
pub fn add_collector(&mut self, collector: Arc<dyn LogCollector + Send + Sync>) {
|
|
self.collectors.push(collector);
|
|
}
|
|
|
|
/// 添加日志过滤器
|
|
pub fn add_filter(&mut self, filter: Arc<dyn LogFilter + Send + Sync>) {
|
|
self.filters.push(filter);
|
|
}
|
|
|
|
/// 记录日志
|
|
pub async fn log(&self, entry: LogEntry) -> Result<()> {
|
|
// 应用过滤器
|
|
for filter in &self.filters {
|
|
if !filter.filter(&entry) {
|
|
return Ok(());
|
|
}
|
|
}
|
|
|
|
// 更新统计
|
|
self.update_stats(&entry).await;
|
|
|
|
// 添加到缓冲区
|
|
let mut buffer = self.buffer.write().await;
|
|
buffer.add_entry(entry.clone());
|
|
|
|
// 检查是否需要刷新
|
|
if buffer.should_flush(&self.config.buffer_config) {
|
|
drop(buffer);
|
|
self.flush_buffer().await?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// 刷新缓冲区
|
|
pub async fn flush_buffer(&self) -> Result<()> {
|
|
let mut buffer = self.buffer.write().await;
|
|
let entries = buffer.drain_entries();
|
|
drop(buffer);
|
|
|
|
// 发送到所有收集器
|
|
for entry in entries {
|
|
for collector in &self.collectors {
|
|
if let Err(e) = collector.collect(&entry) {
|
|
error!(
|
|
"Log collector {} processing failed: {}",
|
|
collector.name(),
|
|
e
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// 刷新所有收集器
|
|
for collector in &self.collectors {
|
|
if let Err(e) = collector.flush() {
|
|
error!(
|
|
"Log collector {} failed to refresh: {}",
|
|
collector.name(),
|
|
e
|
|
);
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// 更新统计信息
|
|
async fn update_stats(&self, entry: &LogEntry) {
|
|
let mut stats = self.stats.write().await;
|
|
stats.total_logs += 1;
|
|
|
|
let level_key = format!("{:?}", entry.level);
|
|
*stats.logs_by_level.entry(level_key).or_insert(0) += 1;
|
|
|
|
*stats
|
|
.logs_by_module
|
|
.entry(entry.module.clone())
|
|
.or_insert(0) += 1;
|
|
|
|
match entry.level {
|
|
LogLevel::Error => stats.error_count += 1,
|
|
LogLevel::Warn => stats.warning_count += 1,
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
/// 获取统计信息
|
|
pub async fn get_stats(&self) -> LoggingStats {
|
|
self.stats.read().await.clone()
|
|
}
|
|
|
|
/// 启动后台任务
|
|
pub async fn start_background_tasks(&self) {
|
|
let buffer = Arc::clone(&self.buffer);
|
|
let config = self.config.clone();
|
|
let logger = self.clone();
|
|
|
|
tokio::spawn(async move {
|
|
let mut interval = tokio::time::interval(config.buffer_config.flush_interval);
|
|
|
|
loop {
|
|
interval.tick().await;
|
|
|
|
let should_flush = {
|
|
let buffer = buffer.read().await;
|
|
!buffer.entries.is_empty()
|
|
};
|
|
|
|
if should_flush {
|
|
if let Err(e) = logger.flush_buffer().await {
|
|
error!("Failed to refresh the log buffer regularly: {}", e);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
impl LogCollector for ConsoleCollector {
|
|
fn collect(&self, entry: &LogEntry) -> Result<()> {
|
|
let formatted = self.formatter.format(entry)?;
|
|
println!("{formatted}");
|
|
Ok(())
|
|
}
|
|
|
|
fn flush(&self) -> Result<()> {
|
|
// 控制台不需要刷新
|
|
Ok(())
|
|
}
|
|
|
|
fn name(&self) -> &str {
|
|
&self.name
|
|
}
|
|
}
|
|
|
|
impl LogCollector for FileCollector {
|
|
fn collect(&self, entry: &LogEntry) -> Result<()> {
|
|
let _formatted = self.formatter.format(entry)?;
|
|
// 这里应该实现文件写入逻辑
|
|
// 包括轮转检查
|
|
Ok(())
|
|
}
|
|
|
|
fn flush(&self) -> Result<()> {
|
|
// 刷新文件缓冲区
|
|
Ok(())
|
|
}
|
|
|
|
fn name(&self) -> &str {
|
|
&self.name
|
|
}
|
|
}
|
|
|
|
impl LogCollector for RemoteCollector {
|
|
fn collect(&self, _entry: &LogEntry) -> Result<()> {
|
|
// 这里应该实现异步发送到远程服务
|
|
// 可以使用队列缓冲
|
|
Ok(())
|
|
}
|
|
|
|
fn flush(&self) -> Result<()> {
|
|
// 刷新远程队列
|
|
Ok(())
|
|
}
|
|
|
|
fn name(&self) -> &str {
|
|
&self.name
|
|
}
|
|
}
|
|
|
|
impl LogFormatter for JsonFormatter {
|
|
fn format(&self, entry: &LogEntry) -> Result<String> {
|
|
serde_json::to_string(entry).context("序列化日志条目为 JSON 失败")
|
|
}
|
|
}
|
|
|
|
impl LogFormatter for TextFormatter {
|
|
fn format(&self, entry: &LogEntry) -> Result<String> {
|
|
let timestamp = entry
|
|
.timestamp
|
|
.duration_since(SystemTime::UNIX_EPOCH)
|
|
.unwrap_or_default();
|
|
|
|
Ok(format!(
|
|
"[{}] {:5} {}: {}",
|
|
timestamp.as_secs(),
|
|
format!("{:?}", entry.level),
|
|
entry.module,
|
|
entry.message
|
|
))
|
|
}
|
|
}
|
|
|
|
impl LogFilter for LevelFilter {
|
|
fn filter(&self, entry: &LogEntry) -> bool {
|
|
self.level_to_number(&entry.level) >= self.level_to_number(&self.min_level)
|
|
}
|
|
|
|
fn name(&self) -> &str {
|
|
"level_filter"
|
|
}
|
|
}
|
|
|
|
impl LevelFilter {
|
|
fn level_to_number(&self, level: &LogLevel) -> u8 {
|
|
match level {
|
|
LogLevel::Trace => 0,
|
|
LogLevel::Debug => 1,
|
|
LogLevel::Info => 2,
|
|
LogLevel::Warn => 3,
|
|
LogLevel::Error => 4,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl LogFilter for ModuleFilter {
|
|
fn filter(&self, entry: &LogEntry) -> bool {
|
|
if !self.denied_modules.is_empty()
|
|
&& self
|
|
.denied_modules
|
|
.iter()
|
|
.any(|m| entry.module.starts_with(m))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if !self.allowed_modules.is_empty() {
|
|
return self
|
|
.allowed_modules
|
|
.iter()
|
|
.any(|m| entry.module.starts_with(m));
|
|
}
|
|
|
|
true
|
|
}
|
|
|
|
fn name(&self) -> &str {
|
|
"module_filter"
|
|
}
|
|
}
|
|
|
|
impl LogFilter for RateLimitFilter {
|
|
fn filter(&self, _entry: &LogEntry) -> bool {
|
|
// 这里应该实现速率限制逻辑
|
|
// 基于模块或其他标识符
|
|
true
|
|
}
|
|
|
|
fn name(&self) -> &str {
|
|
"rate_limit_filter"
|
|
}
|
|
}
|
|
|
|
impl LogBuffer {
|
|
/// 创建新的日志缓冲区
|
|
pub fn new(max_size: usize) -> Self {
|
|
Self {
|
|
entries: Vec::with_capacity(max_size),
|
|
max_size,
|
|
last_flush: SystemTime::now(),
|
|
}
|
|
}
|
|
|
|
/// 添加日志条目
|
|
pub fn add_entry(&mut self, entry: LogEntry) {
|
|
if self.entries.len() >= self.max_size {
|
|
// 移除最旧的条目
|
|
self.entries.remove(0);
|
|
}
|
|
self.entries.push(entry);
|
|
}
|
|
|
|
/// 检查是否应该刷新
|
|
pub fn should_flush(&self, config: &BufferConfig) -> bool {
|
|
self.entries.len() >= config.batch_size
|
|
|| self.last_flush.elapsed().unwrap_or_default() >= config.flush_interval
|
|
}
|
|
|
|
/// 排空缓冲区
|
|
pub fn drain_entries(&mut self) -> Vec<LogEntry> {
|
|
self.last_flush = SystemTime::now();
|
|
std::mem::take(&mut self.entries)
|
|
}
|
|
}
|
|
|
|
impl Default for LoggingConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
level: LogLevel::Info,
|
|
format: LogFormat::Json,
|
|
targets: vec![LogTarget::Console],
|
|
buffer_config: BufferConfig {
|
|
size: 1000,
|
|
flush_interval: Duration::from_secs(5),
|
|
batch_size: 100,
|
|
enable_compression: false,
|
|
},
|
|
rotation_config: RotationConfig {
|
|
max_file_size_mb: 100,
|
|
max_files: 10,
|
|
rotation_interval: Duration::from_secs(3600),
|
|
compress_old_files: true,
|
|
},
|
|
sampling_config: SamplingConfig {
|
|
enabled: false,
|
|
rate: 1.0,
|
|
strategy: SamplingStrategy::Random,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for LoggingStats {
|
|
fn default() -> Self {
|
|
Self {
|
|
total_logs: 0,
|
|
logs_by_level: HashMap::new(),
|
|
logs_by_module: HashMap::new(),
|
|
error_count: 0,
|
|
warning_count: 0,
|
|
avg_processing_time: Duration::from_millis(0),
|
|
buffer_usage: 0.0,
|
|
dropped_logs: 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[tokio::test]
|
|
async fn test_production_logger() {
|
|
let config = LoggingConfig::default();
|
|
let logger = ProductionLogger::new(config);
|
|
|
|
let entry = LogEntry {
|
|
timestamp: SystemTime::now(),
|
|
level: LogLevel::Info,
|
|
message: "测试日志".to_string(),
|
|
module: "test".to_string(),
|
|
file: None,
|
|
line: None,
|
|
fields: HashMap::new(),
|
|
trace_id: None,
|
|
span_id: None,
|
|
};
|
|
|
|
logger.log(entry).await.unwrap();
|
|
|
|
let stats = logger.get_stats().await;
|
|
assert_eq!(stats.total_logs, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn test_level_filter() {
|
|
let filter = LevelFilter {
|
|
min_level: LogLevel::Warn,
|
|
};
|
|
|
|
let info_entry = LogEntry {
|
|
timestamp: SystemTime::now(),
|
|
level: LogLevel::Info,
|
|
message: "信息日志".to_string(),
|
|
module: "test".to_string(),
|
|
file: None,
|
|
line: None,
|
|
fields: HashMap::new(),
|
|
trace_id: None,
|
|
span_id: None,
|
|
};
|
|
|
|
let error_entry = LogEntry {
|
|
level: LogLevel::Error,
|
|
..info_entry.clone()
|
|
};
|
|
|
|
assert!(!filter.filter(&info_entry));
|
|
assert!(filter.filter(&error_entry));
|
|
}
|
|
|
|
#[test]
|
|
fn test_json_formatter() {
|
|
let formatter = JsonFormatter;
|
|
|
|
let entry = LogEntry {
|
|
timestamp: SystemTime::now(),
|
|
level: LogLevel::Info,
|
|
message: "测试消息".to_string(),
|
|
module: "test".to_string(),
|
|
file: Some("test.rs".to_string()),
|
|
line: Some(42),
|
|
fields: HashMap::new(),
|
|
trace_id: Some("trace123".to_string()),
|
|
span_id: Some("span456".to_string()),
|
|
};
|
|
|
|
let formatted = formatter.format(&entry).unwrap();
|
|
assert!(formatted.contains("测试消息"));
|
|
assert!(formatted.contains("trace123"));
|
|
}
|
|
}
|