Egalware/mapo-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://gitlab.steamware.net/steamware/mapo-core into develop

Browse Source
This commit is contained in:
Samuele Locatelli
2026-04-07 12:30:57 +02:00
parent 4676f96889 5d23e63512
commit 45acc78dc9
1 changed files with 152 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MP.IOC/Services/MetricsDbFlushService.cs
+152
View File
@@ -35,6 +35,8 @@ namespace MP.IOC.Services
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await RecoverHistoricalMetricsAsync();
var interval = _config.GetValue<int>("RouteMan:FlushIntervalSeconds", FlushIntervalSeconds);
while (!stoppingToken.IsCancellationRequested)
@@ -55,6 +57,156 @@ namespace MP.IOC.Services
}
}
public async Task RecoverHistoricalMetricsAsync()
{
try
{
var deleteConfirmed = _config.GetValue<bool>("RouteMan:RecoverHistoricalMetrics", false);
if (!deleteConfirmed)
{
Log.Debug("Historical metrics recovery disabled, skipping");
return;
}
Log.Info("Starting historical metrics recovery from Redis...");
var baseKey = _config.GetValue<string>("ServerConf:RedisBaseKey") ?? "MP_IOC";
var lastHour = DateTime.UtcNow.AddHours(-1);
var yesterday = DateTime.UtcNow.AddDays(-1).Date;
var methods = await GetAllMethodsAsync();
foreach (var method in methods)
{
var hoursIndexKey = $"{baseKey}:stats:hours:{method}";
var daysIndexKey = $"{baseKey}:stats:days:{method}";
if (!_db.KeyExists(hoursIndexKey) && !_db.KeyExists(daysIndexKey)) continue;
await RecoverHistoricalHourlyAsync(method, hoursIndexKey, lastHour);
await RecoverHistoricalDailyAsync(method, daysIndexKey, yesterday);
}
Log.Info("Historical metrics recovery completed");
}
catch (Exception ex)
{
Log.Error(ex, "Error recovering historical metrics from Redis");
}
}
private async Task RecoverHistoricalHourlyAsync(string method, string hoursIndexKey, DateTime cutoff)
{
try
{
var allKeys = await _db.SortedSetRangeByRankAsync(hoursIndexKey, 0, -1);
var recordsToInsert = new List<StatsAggregatedModel>();
foreach (var key in allKeys)
{
if (!key.HasValue) continue;
var parts = key.ToString().Split(':');
if (parts.Length >= 4 && DateTime.TryParseExact(parts[3], "yyyyMMddHH",
CultureInfo.InvariantCulture, DateTimeStyles.None, out var hourFromKey))
{
if (hourFromKey <= cutoff)
{
var score = await _db.SortedSetScoreAsync(hoursIndexKey, key);
if (score.HasValue)
{
recordsToInsert.Add(new StatsAggregatedModel
{
Hour = hourFromKey.Date.AddHours(hourFromKey.Hour),
Type = method,
RequestCount = (long)score.Value,
AvgDuration = 0,
MinDuration = 0,
MaxDuration = 0,
NoReply = 0
});
}
}
}
}
if (recordsToInsert.Count > 0)
{
await _aggrService.UpsertManyAsync(recordsToInsert, false);
Log.Info("[HISTORICAL] Recovered {count} hourly records for method {method}", recordsToInsert.Count, method);
var sampleCount = Math.Min(5, recordsToInsert.Count);
for (int i = 0; i < sampleCount; i++)
{
var r = recordsToInsert[i];
Log.Info("[HISTORICAL SAMPLE] Hour={hour}, Type={type}, Count={count}",
r.Hour.ToString("yyyy-MM-dd HH:mm"), r.Type, r.RequestCount);
}
}
}
catch (Exception ex)
{
Log.Error(ex, "Error recovering historical hourly data for method {method}", method);
}
}
private async Task RecoverHistoricalDailyAsync(string method, string daysIndexKey, DateTime cutoff)
{
try
{
var allKeys = await _db.SortedSetRangeByRankAsync(daysIndexKey, 0, -1);
var recordsToInsert = new List<StatsDetailModel>();
foreach (var key in allKeys)
{
if (!key.HasValue) continue;
var parts = key.ToString().Split(':');
if (parts.Length >= 4 && DateTime.TryParseExact(parts[3], "yyyyMMdd",
CultureInfo.InvariantCulture, DateTimeStyles.None, out var dayFromKey))
{
if (dayFromKey.Date <= cutoff.Date)
{
var score = await _db.SortedSetScoreAsync(daysIndexKey, key);
if (score.HasValue)
{
recordsToInsert.Add(new StatsDetailModel
{
Environment = parts.Length > 4 ? parts[4] : "unknown",
Type = method,
Hour = dayFromKey.Date,
RequestCount = (long)score.Value,
AvgDuration = 0,
MinDuration = 0,
MaxDuration = 0,
NoReply = 0
});
}
}
}
}
if (recordsToInsert.Count > 0)
{
await _detailService.UpsertManyAsync(recordsToInsert, false);
Log.Info("[HISTORICAL] Recovered {count} daily records for method {method}", recordsToInsert.Count, method);
var sampleCount = Math.Min(5, recordsToInsert.Count);
for (int i = 0; i < sampleCount; i++)
{
var r = recordsToInsert[i];
Log.Info("[HISTORICAL SAMPLE] Env={env}, Type={type}, Date={date}, Count={count}",
r.Environment, r.Type, r.Hour.ToString("yyyy-MM-dd"), r.RequestCount);
}
}
}
catch (Exception ex)
{
Log.Error(ex, "Error recovering historical daily data for method {method}", method);
}
}
public async Task FlushMetricsAsync()
{
var utcNow = DateTime.Now;