diff --git a/MP.IOC/Services/MetricsDbFlushService.cs b/MP.IOC/Services/MetricsDbFlushService.cs index f238e4ff..66e05e77 100644 --- a/MP.IOC/Services/MetricsDbFlushService.cs +++ b/MP.IOC/Services/MetricsDbFlushService.cs @@ -35,6 +35,8 @@ namespace MP.IOC.Services protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + await RecoverHistoricalMetricsAsync(); + var interval = _config.GetValue("RouteMan:FlushIntervalSeconds", FlushIntervalSeconds); while (!stoppingToken.IsCancellationRequested) @@ -55,6 +57,156 @@ namespace MP.IOC.Services } } + public async Task RecoverHistoricalMetricsAsync() + { + try + { + var deleteConfirmed = _config.GetValue("RouteMan:RecoverHistoricalMetrics", false); + + if (!deleteConfirmed) + { + Log.Debug("Historical metrics recovery disabled, skipping"); + return; + } + + Log.Info("Starting historical metrics recovery from Redis..."); + + var baseKey = _config.GetValue("ServerConf:RedisBaseKey") ?? "MP_IOC"; + var lastHour = DateTime.UtcNow.AddHours(-1); + var yesterday = DateTime.UtcNow.AddDays(-1).Date; + + var methods = await GetAllMethodsAsync(); + + foreach (var method in methods) + { + var hoursIndexKey = $"{baseKey}:stats:hours:{method}"; + var daysIndexKey = $"{baseKey}:stats:days:{method}"; + + if (!_db.KeyExists(hoursIndexKey) && !_db.KeyExists(daysIndexKey)) continue; + + await RecoverHistoricalHourlyAsync(method, hoursIndexKey, lastHour); + await RecoverHistoricalDailyAsync(method, daysIndexKey, yesterday); + } + + Log.Info("Historical metrics recovery completed"); + } + catch (Exception ex) + { + Log.Error(ex, "Error recovering historical metrics from Redis"); + } + } + + private async Task RecoverHistoricalHourlyAsync(string method, string hoursIndexKey, DateTime cutoff) + { + try + { + var allKeys = await _db.SortedSetRangeByRankAsync(hoursIndexKey, 0, -1); + var recordsToInsert = new List(); + + foreach (var key in allKeys) + { + if (!key.HasValue) continue; + + var parts = key.ToString().Split(':'); + if (parts.Length >= 4 && DateTime.TryParseExact(parts[3], "yyyyMMddHH", + CultureInfo.InvariantCulture, DateTimeStyles.None, out var hourFromKey)) + { + if (hourFromKey <= cutoff) + { + var score = await _db.SortedSetScoreAsync(hoursIndexKey, key); + if (score.HasValue) + { + recordsToInsert.Add(new StatsAggregatedModel + { + Hour = hourFromKey.Date.AddHours(hourFromKey.Hour), + Type = method, + RequestCount = (long)score.Value, + AvgDuration = 0, + MinDuration = 0, + MaxDuration = 0, + NoReply = 0 + }); + } + } + } + } + + if (recordsToInsert.Count > 0) + { + await _aggrService.UpsertManyAsync(recordsToInsert, false); + Log.Info("[HISTORICAL] Recovered {count} hourly records for method {method}", recordsToInsert.Count, method); + + var sampleCount = Math.Min(5, recordsToInsert.Count); + for (int i = 0; i < sampleCount; i++) + { + var r = recordsToInsert[i]; + Log.Info("[HISTORICAL SAMPLE] Hour={hour}, Type={type}, Count={count}", + r.Hour.ToString("yyyy-MM-dd HH:mm"), r.Type, r.RequestCount); + } + } + } + catch (Exception ex) + { + Log.Error(ex, "Error recovering historical hourly data for method {method}", method); + } + } + + private async Task RecoverHistoricalDailyAsync(string method, string daysIndexKey, DateTime cutoff) + { + try + { + var allKeys = await _db.SortedSetRangeByRankAsync(daysIndexKey, 0, -1); + var recordsToInsert = new List(); + + foreach (var key in allKeys) + { + if (!key.HasValue) continue; + + var parts = key.ToString().Split(':'); + if (parts.Length >= 4 && DateTime.TryParseExact(parts[3], "yyyyMMdd", + CultureInfo.InvariantCulture, DateTimeStyles.None, out var dayFromKey)) + { + if (dayFromKey.Date <= cutoff.Date) + { + var score = await _db.SortedSetScoreAsync(daysIndexKey, key); + if (score.HasValue) + { + recordsToInsert.Add(new StatsDetailModel + { + Environment = parts.Length > 4 ? parts[4] : "unknown", + Type = method, + Hour = dayFromKey.Date, + RequestCount = (long)score.Value, + AvgDuration = 0, + MinDuration = 0, + MaxDuration = 0, + NoReply = 0 + }); + } + } + } + } + + if (recordsToInsert.Count > 0) + { + await _detailService.UpsertManyAsync(recordsToInsert, false); + Log.Info("[HISTORICAL] Recovered {count} daily records for method {method}", recordsToInsert.Count, method); + + var sampleCount = Math.Min(5, recordsToInsert.Count); + for (int i = 0; i < sampleCount; i++) + { + var r = recordsToInsert[i]; + Log.Info("[HISTORICAL SAMPLE] Env={env}, Type={type}, Date={date}, Count={count}", + r.Environment, r.Type, r.Hour.ToString("yyyy-MM-dd"), r.RequestCount); + } + } + } + catch (Exception ex) + { + Log.Error(ex, "Error recovering historical daily data for method {method}", method); + } + } + public async Task FlushMetricsAsync() { var utcNow = DateTime.Now;