Samuele Locatelli
1017 lines
38 KiB
C#
1017 lines
38 KiB
C#
using EgwCoreLib.Lux.Core.Stats;
|
||
using EgwCoreLib.Lux.Data.Controllers;
|
||
using EgwCoreLib.Lux.Data.DbModel.Stats;
|
||
using Microsoft.EntityFrameworkCore;
|
||
using Microsoft.Extensions.Configuration;
|
||
using NLog;
|
||
using StackExchange.Redis;
|
||
using System;
|
||
using System.Collections.Generic;
|
||
using System.Linq;
|
||
using System.Text;
|
||
using System.Threading.Tasks;
|
||
using static Egw.Window.Data.Enums;
|
||
|
||
namespace EgwCoreLib.Lux.Data.Services
|
||
{
|
||
/// <summary>
|
||
/// Gestione servizio indice richieste
|
||
/// </summary>
|
||
public class CalcRuidService : BaseServ
|
||
{
|
||
#region Public Constructors
|
||
|
||
public CalcRuidService(IConfiguration configuration, IConnectionMultiplexer redisConn, string redisBaseKey, TimeSpan retention, TimeSpan archivePeriod) : base(configuration, redisConn)
|
||
{
|
||
_db = redisConn.GetDatabase();
|
||
_base = redisBaseKey.TrimEnd(':');
|
||
_retention = retention;
|
||
_archivePeriod = archivePeriod;
|
||
// conf DB
|
||
string connStr = BaseServ._config.GetConnectionString("Lux.All") ?? "";
|
||
if (string.IsNullOrEmpty(connStr))
|
||
{
|
||
Log.Error("ConnString empty!");
|
||
}
|
||
else
|
||
{
|
||
//dbController = new Controllers.LuxController(_config);
|
||
dbController = new LuxController();
|
||
StringBuilder sb = new StringBuilder();
|
||
sb.AppendLine($"CalcRuidService | LuxController OK");
|
||
Log.Info(sb.ToString());
|
||
}
|
||
}
|
||
|
||
#endregion Public Constructors
|
||
|
||
#region Public Enums
|
||
|
||
/// <summary>
|
||
/// Modalità raggruppamento (giornalieri, orari...)
|
||
/// </summary>
|
||
public enum GroupMode
|
||
{
|
||
Day,
|
||
Hour
|
||
}
|
||
|
||
/// <summary>
|
||
/// Tipo di dati raggruppamento gestiti
|
||
/// </summary>
|
||
public enum TagMode
|
||
{
|
||
Envir,
|
||
Mode
|
||
}
|
||
|
||
#endregion Public Enums
|
||
|
||
#region Public Methods
|
||
|
||
/// <summary>
|
||
/// Metodo Creazione nuova richiesta
|
||
/// </summary>
|
||
/// <param name="envir">Environment calcolo</param>
|
||
/// <param name="tipo">Tipologia richiesta</param>
|
||
/// <param name="uid">UID di riferimento</param>
|
||
/// <returns>restituisce il valore del RUID (ID univoco richiesta)</returns>
|
||
public async Task<string> AddRequestAsync(string envir, string tipo, string uid)
|
||
{
|
||
var ruid = GenerateRuid();
|
||
var processStart = DateTimeOffset.Now.ToUnixTimeMilliseconds();
|
||
|
||
var hashKey = GetRequestKey(ruid);
|
||
var setKey = GetSortedSetKey(envir, tipo);
|
||
var uidKey = GetUidSetKey(uid);
|
||
var combKey = GetCombinationsKey();
|
||
|
||
var batch = _db.CreateBatch();
|
||
|
||
// Info di base sulla richiesta
|
||
var t1 = batch.HashSetAsync(hashKey, new HashEntry[]
|
||
{
|
||
new HashEntry("processStart", processStart),
|
||
new HashEntry("UID", uid),
|
||
new HashEntry("tipo", tipo),
|
||
new HashEntry("envir", envir)
|
||
});
|
||
|
||
var t2 = batch.SortedSetAddAsync(setKey, ruid, processStart);
|
||
var t3 = batch.SetAddAsync(uidKey, ruid);
|
||
|
||
string comb = $"{envir}|{tipo}";
|
||
var t4 = batch.SetAddAsync(combKey, comb);
|
||
|
||
// --- Nuova logica: un unico hash per slot orario ---
|
||
string hourKeyStr = $"stats:requests:{DateTime.Now:yyyyMMddHH}";
|
||
RedisKey hourKey = Key(hourKeyStr);
|
||
|
||
int minute = DateTime.Now.Minute;
|
||
string minuteField = minute.ToString("D2"); // "00".."59"
|
||
|
||
// Incremento totale e minuto corrente
|
||
var t5 = batch.HashIncrementAsync(hourKey, "tot", 1);
|
||
var t6 = batch.HashIncrementAsync(hourKey, minuteField, 1);
|
||
|
||
// Esegue il batch
|
||
batch.Execute();
|
||
|
||
// Attendo completamento
|
||
await Task.WhenAll(t1, t2, t3, t4, t5, t6);
|
||
|
||
return ruid;
|
||
}
|
||
|
||
#if false
|
||
/// <summary>
|
||
/// Metodo di Cleanup periodico
|
||
/// </summary>
|
||
/// <param name="environment">Environment calcolo</param>
|
||
/// <param name="tipo">Tipologia richiesta</param>
|
||
/// <returns></returns>
|
||
public async Task CleanupOldRequestsAsync(string environment, string tipo)
|
||
{
|
||
var cutoff = DateTimeOffset.Now.Add(-_retention).ToUnixTimeMilliseconds();
|
||
var setKey = GetSortedSetKey(environment, tipo);
|
||
|
||
var oldIds = await _db.SortedSetRangeByScoreAsync(setKey, stop: cutoff);
|
||
if (oldIds.Length == 0) return;
|
||
|
||
var batch = _db.CreateBatch();
|
||
var tasks = new List<Task>();
|
||
|
||
foreach (var id in oldIds)
|
||
{
|
||
var ruid = id.ToString();
|
||
var hashKey = GetRequestKey(ruid);
|
||
|
||
var uid = await _db.HashGetAsync(hashKey, "UID");
|
||
if (!uid.IsNull)
|
||
{
|
||
var uidKey = GetUidSetKey(uid);
|
||
tasks.Add(batch.SetRemoveAsync(uidKey, ruid));
|
||
|
||
tasks.Add(batch.SetLengthAsync(uidKey).ContinueWith(t =>
|
||
{
|
||
if (t.Result == 0)
|
||
_db.KeyDelete(uidKey);
|
||
}));
|
||
}
|
||
|
||
tasks.Add(batch.KeyDeleteAsync(hashKey));
|
||
}
|
||
|
||
tasks.Add(batch.SortedSetRemoveRangeByScoreAsync(setKey, double.NegativeInfinity, cutoff));
|
||
|
||
tasks.Add(batch.SortedSetLengthAsync(setKey).ContinueWith(t =>
|
||
{
|
||
if (t.Result == 0)
|
||
_db.KeyDelete(setKey);
|
||
}));
|
||
|
||
batch.Execute();
|
||
await Task.WhenAll(tasks);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Metodo di Cleanup dato periodo limite prima del quale va eliminato l'elenco dei dati
|
||
/// </summary>
|
||
/// <param name="environment">Environment calcolo</param>
|
||
/// <param name="tipo">Tipologia richiesta</param>
|
||
/// <param name="timeLimit">Limite temporale dati da eliminare</param>
|
||
/// <returns></returns>
|
||
public async Task CleanupOldRequestsAsync(string environment, string tipo, TimeSpan timeLimit)
|
||
{
|
||
var cutoff = DateTimeOffset.Now.Add(-timeLimit).ToUnixTimeMilliseconds();
|
||
var setKey = GetSortedSetKey(environment, tipo);
|
||
|
||
var oldIds = await _db.SortedSetRangeByScoreAsync(setKey, stop: cutoff);
|
||
if (oldIds.Length == 0) return;
|
||
|
||
var batch = _db.CreateBatch();
|
||
var tasks = new List<Task>();
|
||
|
||
foreach (var id in oldIds)
|
||
{
|
||
var ruid = id.ToString();
|
||
var hashKey = GetRequestKey(ruid);
|
||
|
||
var uid = await _db.HashGetAsync(hashKey, "UID");
|
||
if (!uid.IsNull)
|
||
{
|
||
var uidKey = GetUidSetKey(uid);
|
||
tasks.Add(batch.SetRemoveAsync(uidKey, ruid));
|
||
|
||
tasks.Add(batch.SetLengthAsync(uidKey).ContinueWith(t =>
|
||
{
|
||
if (t.Result == 0)
|
||
_db.KeyDelete(uidKey);
|
||
}));
|
||
}
|
||
|
||
tasks.Add(batch.KeyDeleteAsync(hashKey));
|
||
}
|
||
|
||
tasks.Add(batch.SortedSetRemoveRangeByScoreAsync(setKey, double.NegativeInfinity, cutoff));
|
||
|
||
tasks.Add(batch.SortedSetLengthAsync(setKey).ContinueWith(t =>
|
||
{
|
||
if (t.Result == 0)
|
||
_db.KeyDelete(setKey);
|
||
}));
|
||
|
||
batch.Execute();
|
||
await Task.WhenAll(tasks);
|
||
}
|
||
#endif
|
||
|
||
/// <summary>
|
||
/// Effettua pulizia delle statistiche collezionate una volta migrate sul DB
|
||
/// </summary>
|
||
/// <param name="keysToDelete"></param>
|
||
/// <returns></returns>
|
||
public async Task CleanupStatsAsync(List<RedisValue> keysToDelete)
|
||
{
|
||
if (keysToDelete == null || !keysToDelete.Any()) return;
|
||
|
||
var batch = _db.CreateBatch();
|
||
var tasks = new List<Task>();
|
||
|
||
foreach (var hourStr in keysToDelete)
|
||
{
|
||
// Costruisco le chiavi Redis da cancellare
|
||
var aggKeyStr = $"stats:agg:{hourStr}";
|
||
var reqKeyStr = $"stats:requests:{hourStr}";
|
||
|
||
// Cancello hash delle statistiche aggregate
|
||
tasks.Add(batch.KeyDeleteAsync(Key(aggKeyStr)));
|
||
// Rimuovo dall’indice
|
||
tasks.Add(batch.SetRemoveAsync(Key("stats:agg:index"), (RedisValue)$"{hourStr}"));
|
||
|
||
// Cancello hash delle richieste per la stessa ora
|
||
tasks.Add(batch.KeyDeleteAsync(Key(reqKeyStr)));
|
||
}
|
||
|
||
// Pulizia delle richieste vecchie () ora precedente) per tutte le combinazioni env/tipo
|
||
DateTime adesso = DateTime.Now;
|
||
var cutoff = DateTimeOffset.Now.AddMinutes(-adesso.Minute).AddHours(-1).ToUnixTimeMilliseconds();
|
||
//var dtFrom = DateTimeOffset.Now.Add(-_retention).ToUnixTimeMilliseconds();
|
||
var combos = await GetCombinationsAsync();
|
||
|
||
foreach (var (env, tipo) in combos)
|
||
{
|
||
var setKey = GetSortedSetKey(env, tipo);
|
||
|
||
var oldIds = await _db.SortedSetRangeByScoreAsync(setKey, stop: cutoff);
|
||
foreach (var id in oldIds)
|
||
{
|
||
var ruid = id.ToString();
|
||
var hashKey = GetRequestKey(ruid);
|
||
|
||
var uid = await _db.HashGetAsync(hashKey, "UID");
|
||
if (!uid.IsNull)
|
||
{
|
||
var uidKey = GetUidSetKey(uid);
|
||
tasks.Add(batch.SetRemoveAsync(uidKey, ruid));
|
||
|
||
tasks.Add(batch.SetLengthAsync(uidKey).ContinueWith(t =>
|
||
{
|
||
if (t.Result == 0)
|
||
_db.KeyDelete(uidKey);
|
||
}));
|
||
}
|
||
|
||
tasks.Add(batch.KeyDeleteAsync(hashKey));
|
||
}
|
||
|
||
tasks.Add(batch.SortedSetRemoveRangeByScoreAsync(setKey, double.NegativeInfinity, cutoff));
|
||
tasks.Add(batch.SortedSetLengthAsync(setKey).ContinueWith(t =>
|
||
{
|
||
if (t.Result == 0)
|
||
_db.KeyDelete(setKey);
|
||
}));
|
||
}
|
||
|
||
batch.Execute();
|
||
await Task.WhenAll(tasks);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Metodo di Aggiornamento richiesta esistente
|
||
/// </summary>
|
||
/// <param name="ruid">RUID richiesta</param>
|
||
/// <returns></returns>
|
||
public async Task CompleteRequestAsync(string ruid)
|
||
{
|
||
var hashKey = GetRequestKey(ruid);
|
||
DateTime adesso = DateTime.Now;
|
||
var processEnd = DateTimeOffset.Now.ToUnixTimeMilliseconds();
|
||
|
||
// Recupero processStart e altri dati
|
||
var processStartValue = await _db.HashGetAsync(hashKey, "processStart");
|
||
if (processStartValue.IsNull) return;
|
||
var envir = await _db.HashGetAsync(hashKey, "envir");
|
||
var tipo = await _db.HashGetAsync(hashKey, "tipo");
|
||
|
||
var processStart = (long)processStartValue;
|
||
var elapsed = (processEnd - processStart) / 1000.0; // secondi con decimali
|
||
|
||
// Aggiorno hash della richiesta
|
||
await _db.HashSetAsync(hashKey, new HashEntry[]
|
||
{
|
||
new HashEntry("processEnd", processEnd),
|
||
new HashEntry("processElapsed", elapsed)
|
||
});
|
||
|
||
// Chiave aggregata per slot orario
|
||
string dayKey = $"{adesso:yyyyMMdd}";
|
||
string dayHourKey = $"{adesso:yyyyMMddHH}";
|
||
RedisKey aggHourKey = Key($"stats:agg:{dayHourKey}");
|
||
RedisKey rtEnvDayKey = Key($"stats:rtime:env:{dayKey}");
|
||
RedisKey rtEnvHourKey = Key($"stats:rtime:env:{dayHourKey}");
|
||
RedisKey rtModeDayKey = Key($"stats:rtime:mode:{dayKey}");
|
||
RedisKey rtModeHourKey = Key($"stats:rtime:mode:{dayHourKey}");
|
||
await _db.SetAddAsync(Key("stats:agg:index"), dayHourKey);
|
||
|
||
// --- Aggiornamento globale (ALL|ALL) ---
|
||
var globalKey = "ALL|ALL";
|
||
|
||
// Somma e count aggregati
|
||
var tasks = new List<Task>
|
||
{
|
||
_db.HashIncrementAsync(aggHourKey, $"{globalKey}:sum", elapsed),
|
||
_db.HashIncrementAsync(aggHourKey, $"{globalKey}:count", 1)
|
||
};
|
||
|
||
// sommo durata e
|
||
|
||
// Max globale
|
||
var currentGlobalMax = await _db.HashGetAsync(aggHourKey, $"{globalKey}:max");
|
||
double maxGlobal = currentGlobalMax.IsNull ? 0 : (double)currentGlobalMax;
|
||
if (elapsed > maxGlobal)
|
||
{
|
||
tasks.Add(_db.HashSetAsync(aggHourKey, new HashEntry[]
|
||
{
|
||
new HashEntry($"{globalKey}:max", elapsed)
|
||
}));
|
||
}
|
||
|
||
// --- Aggiornamento per env/tipo ---
|
||
if (!envir.IsNull && !tipo.IsNull)
|
||
{
|
||
var envTipo = $"{envir}|{tipo}";
|
||
string sTipo = $"{tipo}";
|
||
var mode = sTipo.Substring(0, sTipo.IndexOf('-'));
|
||
|
||
// dati statistiche da storicizzare
|
||
tasks.Add(_db.HashIncrementAsync(aggHourKey, $"{envTipo}:sum", elapsed));
|
||
tasks.Add(_db.HashIncrementAsync(aggHourKey, $"{envTipo}:count", 1));
|
||
|
||
// dati RT giornalieri/orari x ENV
|
||
tasks.Add(_db.HashIncrementAsync(rtEnvDayKey, $"{envir}:sum", elapsed));
|
||
tasks.Add(_db.HashIncrementAsync(rtEnvDayKey, $"{envir}:count", 1));
|
||
tasks.Add(_db.HashIncrementAsync(rtEnvHourKey, $"{envir}:sum", elapsed));
|
||
tasks.Add(_db.HashIncrementAsync(rtEnvHourKey, $"{envir}:count", 1));
|
||
|
||
// dati RT giornalieri/orari per MODE
|
||
tasks.Add(_db.HashIncrementAsync(rtModeDayKey, $"{mode}:sum", elapsed));
|
||
tasks.Add(_db.HashIncrementAsync(rtModeDayKey, $"{mode}:count", 1));
|
||
tasks.Add(_db.HashIncrementAsync(rtModeHourKey, $"{mode}:sum", elapsed));
|
||
tasks.Add(_db.HashIncrementAsync(rtModeHourKey, $"{mode}:count", 1));
|
||
|
||
// imposto TTL delle hash RealTime...
|
||
tasks.Add(_db.KeyExpireAsync(rtEnvDayKey, TimeSpan.FromDays(8)));
|
||
tasks.Add(_db.KeyExpireAsync(rtEnvHourKey, TimeSpan.FromDays(8)));
|
||
tasks.Add(_db.KeyExpireAsync(rtModeDayKey, TimeSpan.FromDays(8)));
|
||
tasks.Add(_db.KeyExpireAsync(rtModeHourKey, TimeSpan.FromDays(8)));
|
||
|
||
// ora calcolo max e resto
|
||
var currentMaxForGroup = await _db.HashGetAsync(aggHourKey, $"{envTipo}:max");
|
||
double maxGroup = currentMaxForGroup.IsNull ? 0 : (double)currentMaxForGroup;
|
||
if (elapsed > maxGroup)
|
||
{
|
||
tasks.Add(_db.HashSetAsync(aggHourKey, new HashEntry[]
|
||
{
|
||
new HashEntry($"{envTipo}:max", elapsed)
|
||
}));
|
||
}
|
||
}
|
||
|
||
// Attendo completamento di tutti i task
|
||
await Task.WhenAll(tasks);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Espostazione statistiche da REDIS al DB
|
||
/// </summary>
|
||
/// <param name="clearRedisAfterExport"></param>
|
||
/// <returns></returns>
|
||
public async Task ExportStatsToDbAsync(bool clearRedisAfterExport = false)
|
||
{
|
||
var aggregatedData = new List<StatsAggregatedModel>();
|
||
var detailData = new List<StatsDetailModel>();
|
||
|
||
// Recupero tutte le chiavi dall'indice (sono RedisValue)
|
||
RedisValue[] indexValues = await _db.SetMembersAsync(Key("stats:agg:index"));
|
||
|
||
// Ora corrente (slot da saltare IN DELETE)
|
||
var currentHourKey = $"{DateTime.Now:yyyyMMddHH}";
|
||
|
||
// Lista chiavi da cancellare dopo export (uso RedisValue perché l'indice è un set di valori)
|
||
var keysToDelete = new List<RedisValue>();
|
||
|
||
foreach (RedisValue indexValue in indexValues)
|
||
{
|
||
string hourStr = $"{indexValue}";
|
||
var keyStr = Key($"stats:agg:{hourStr}");
|
||
|
||
// Estraggo l'ora dalla chiave
|
||
if (!DateTime.TryParseExact(hourStr, "yyyyMMddHH", null,
|
||
System.Globalization.DateTimeStyles.None, out DateTime hour))
|
||
continue;
|
||
|
||
// Recupero tutti i valori dell'hash (qui serve RedisKey)
|
||
HashEntry[] entries = await _db.HashGetAllAsync(keyStr);
|
||
|
||
// --- Globali ALL|ALL ---
|
||
RedisValue sumGlobal = entries.FirstOrDefault(e => e.Name == "ALL|ALL:sum").Value;
|
||
RedisValue countGlobal = entries.FirstOrDefault(e => e.Name == "ALL|ALL:count").Value;
|
||
RedisValue maxGlobal = entries.FirstOrDefault(e => e.Name == "ALL|ALL:max").Value;
|
||
|
||
if (!sumGlobal.IsNull && !countGlobal.IsNull)
|
||
{
|
||
long count = (long)countGlobal;
|
||
double sum = (double)sumGlobal;
|
||
double max = maxGlobal.IsNull ? 0 : (double)maxGlobal;
|
||
|
||
aggregatedData.Add(new StatsAggregatedModel
|
||
{
|
||
Hour = hour,
|
||
RequestCount = count,
|
||
AvgDuration = count > 0 ? sum / count : 0,
|
||
MaxDuration = max
|
||
});
|
||
}
|
||
|
||
// --- Dettagli per env|tipo ---
|
||
IEnumerable<string> grouped = entries
|
||
.Select(e => e.Name.ToString())
|
||
.Where(n => !n.StartsWith("ALL|ALL"))
|
||
.Select(n => n.Split(':')[0])
|
||
.Distinct();
|
||
|
||
foreach (string envTipo in grouped)
|
||
{
|
||
string[] envTipoSplit = envTipo.Split('|');
|
||
if (envTipoSplit.Length != 2) continue;
|
||
|
||
string env = envTipoSplit[0];
|
||
string tipo = envTipoSplit[1];
|
||
|
||
RedisValue sumVal = entries.FirstOrDefault(e => e.Name == $"{envTipo}:sum").Value;
|
||
RedisValue countVal = entries.FirstOrDefault(e => e.Name == $"{envTipo}:count").Value;
|
||
RedisValue maxVal = entries.FirstOrDefault(e => e.Name == $"{envTipo}:max").Value;
|
||
|
||
if (!sumVal.IsNull && !countVal.IsNull)
|
||
{
|
||
long count = (long)countVal;
|
||
double sum = (double)sumVal;
|
||
double max = maxVal.IsNull ? 0 : (double)maxVal;
|
||
|
||
detailData.Add(new StatsDetailModel
|
||
{
|
||
Environment = env,
|
||
Type = tipo,
|
||
Hour = hour,
|
||
RequestCount = count,
|
||
AvgDuration = count > 0 ? sum / count : 0,
|
||
MaxDuration = max
|
||
});
|
||
}
|
||
}
|
||
|
||
// se ora corrente NON la eliminerò...
|
||
if (hourStr == currentHourKey)
|
||
continue; // salto slot corrente
|
||
// Se richiesto, preparo cancellazione
|
||
if (clearRedisAfterExport)
|
||
{
|
||
keysToDelete.Add(indexValue); // RedisValue coerente
|
||
}
|
||
}
|
||
|
||
// Scrittura su DB
|
||
long aggrResult = await dbController.StatsAggrUpsertAsync(aggregatedData, true);
|
||
long detailResult = await dbController.StatsDetailModelUpsertAsync(detailData, true);
|
||
|
||
// Se entrambe hanno successo, eseguo batch di cancellazione
|
||
if (clearRedisAfterExport && aggrResult > 0 && detailResult > 0 && keysToDelete.Any())
|
||
{
|
||
await CleanupStatsAsync(keysToDelete);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Metodo Recupero combinazioni envir/tipo
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
public async Task<IEnumerable<(string env, string tipo)>> GetCombinationsAsync()
|
||
{
|
||
var members = await _db.SetMembersAsync(GetCombinationsKey());
|
||
return members
|
||
.Select(x => x.ToString().Split('|'))
|
||
.Select(a => (a[0], a[1]));
|
||
}
|
||
|
||
/// <summary>
|
||
/// Gestione statistiche (da rivedere)
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
public async Task<List<StatsDetailModel>> GetOldStatsFromRedisAsync()
|
||
{
|
||
var results = new List<StatsDetailModel>();
|
||
var now = DateTime.Now;
|
||
var cutoff = now.AddHours(-24);
|
||
|
||
for (var dt = cutoff.Date; dt < now; dt = dt.AddHours(1))
|
||
{
|
||
var hourKey = Key($"stats:agg:{dt:yyyyMMddHH}");
|
||
var entries = await _db.HashGetAllAsync(hourKey);
|
||
if (entries.Length == 0) continue;
|
||
|
||
foreach (var entry in entries)
|
||
{
|
||
var key = entry.Name.ToString();
|
||
var value = (double)entry.Value;
|
||
|
||
// Dividi chiave per ottenere envir e tipo
|
||
var parts = key.Split(':');
|
||
if (parts.Length < 2) continue;
|
||
|
||
string groupKey = parts[0]; // es: "prod|api"
|
||
string metric = parts[1]; // es: "count", "sum", "max"
|
||
|
||
// Se è una chiave di tipo "count", "sum", o "max", puoi costruire l'oggetto StatsDetailModel
|
||
if (metric == "count")
|
||
{
|
||
var envTipo = groupKey.Split('|');
|
||
var envir = envTipo[0];
|
||
var tipo = envTipo[1];
|
||
|
||
results.Add(new StatsDetailModel
|
||
{
|
||
Environment = envir,
|
||
Type = tipo,
|
||
Hour = dt,
|
||
RequestCount = (long)value,
|
||
AvgDuration = 0, // Calcolato dopo aggregazione
|
||
MaxDuration = 0 // Calcolato dopo aggregazione
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
return results;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Metodo Recupero richieste per UID
|
||
/// </summary>
|
||
/// <param name="uid"></param>
|
||
/// <returns></returns>
|
||
public async Task<IEnumerable<string>> GetRequestsByUidAsync(string uid)
|
||
{
|
||
var uidKey = GetUidSetKey(uid);
|
||
var members = await _db.SetMembersAsync(uidKey);
|
||
return members.Select(x => x.ToString());
|
||
}
|
||
|
||
/// <summary>
|
||
/// Metodo Statistiche aggregate
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
public async Task<StatsRealtimeDto> GetStatsAsync()
|
||
{
|
||
RedisKey minuteKey = Key($"stats:requests:count:{DateTime.Now:yyyyMMddHHmm}");
|
||
RedisKey hourKey = Key($"stats:requests:count:{DateTime.Now:yyyyMMddHH}");
|
||
RedisKey hourSumKey = Key($"stats:processing:sum:{DateTime.Now:yyyyMMddHH}");
|
||
RedisKey hourMaxKey = Key($"stats:processing:max:{DateTime.Now:yyyyMMddHH}");
|
||
|
||
var minuteCount = await _db.StringGetAsync(minuteKey);
|
||
var hourCount = await _db.StringGetAsync(hourKey);
|
||
var sum = await _db.StringGetAsync(hourSumKey);
|
||
var max = await _db.StringGetAsync(hourMaxKey);
|
||
|
||
long minCnt = minuteCount.IsNull ? 0 : (long)minuteCount;
|
||
long hourCnt = hourCount.IsNull ? 0 : (long)hourCount;
|
||
|
||
double sumVal = sum.IsNull ? 0 : (double)sum;
|
||
double maxVal = max.IsNull ? 0 : (double)max;
|
||
|
||
double avg = hourCnt > 0 ? sumVal / hourCnt : 0;
|
||
|
||
return new StatsRealtimeDto
|
||
{
|
||
RequestsLastMinute = minCnt,
|
||
RequestsLastHour = hourCnt,
|
||
ProcessingAvgLastHour = avg,
|
||
ProcessingMaxLastHour = maxVal
|
||
};
|
||
}
|
||
|
||
/// <summary>
|
||
/// Restituisce le statistiche per range date
|
||
/// </summary>
|
||
/// <param name="from"></param>
|
||
/// <param name="to"></param>
|
||
/// <returns></returns>
|
||
public async Task<StatsRangeDto> GetStatsRangeAsync(DateTime from, DateTime to)
|
||
{
|
||
var dto = new StatsRangeDto();
|
||
|
||
var hourKeys = new List<RedisKey>();
|
||
var sumKeys = new List<RedisKey>();
|
||
var maxKeys = new List<RedisKey>();
|
||
|
||
var cursor = from;
|
||
|
||
while (cursor <= to)
|
||
{
|
||
hourKeys.Add(Key($"stats:requests:count:{cursor:yyyyMMddHH}"));
|
||
sumKeys.Add(Key($"stats:processing:sum:{cursor:yyyyMMddHH}"));
|
||
maxKeys.Add(Key($"stats:processing:max:{cursor:yyyyMMddHH}"));
|
||
|
||
dto.HourLabels.Add(cursor.ToString("dd/MM HH:mm"));
|
||
cursor = cursor.AddHours(1);
|
||
}
|
||
|
||
var batch = _db.CreateBatch();
|
||
|
||
var hourTasks = hourKeys.Select(k => batch.StringGetAsync(k)).ToArray();
|
||
var sumTasks = sumKeys.Select(k => batch.StringGetAsync(k)).ToArray();
|
||
var maxTasks = maxKeys.Select(k => batch.StringGetAsync(k)).ToArray();
|
||
|
||
batch.Execute();
|
||
|
||
await Task.WhenAll(hourTasks);
|
||
await Task.WhenAll(sumTasks);
|
||
await Task.WhenAll(maxTasks);
|
||
|
||
for (int i = 0; i < hourTasks.Length; i++)
|
||
{
|
||
long req = hourTasks[i].Result.IsNull ? 0 : (long)hourTasks[i].Result;
|
||
double sum = sumTasks[i].Result.IsNull ? 0 : (double)sumTasks[i].Result;
|
||
double max = maxTasks[i].Result.IsNull ? 0 : (double)maxTasks[i].Result;
|
||
|
||
dto.Requests.Add(req);
|
||
dto.MaxProcessing.Add(max);
|
||
|
||
double avg = req > 0 ? sum / req : 0;
|
||
dto.AvgProcessing.Add(avg);
|
||
}
|
||
|
||
return dto;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Restituisce le statistiche RealTime da REDIS
|
||
/// </summary>
|
||
/// <param name="reqGroup">Raggruppamento orario richiesto</param>
|
||
/// <param name="reqTag">Tipologia dati richiesti</param>
|
||
/// <param name="dtFrom">DataOra inizio recupero</param>
|
||
/// <returns></returns>
|
||
public async Task<List<RealtimeProcDto>> RealTimeDataStats(GroupMode reqGroup, TagMode reqTag, DateTime dtFrom)
|
||
{
|
||
var adesso = DateTime.Now;
|
||
var tasks = new List<Task<HashEntry[]>>();
|
||
var keys = new List<RedisKey>();
|
||
var dtList = new List<DateTime>();
|
||
|
||
// calcolo step richiesto
|
||
int step = reqGroup == GroupMode.Day ? 24 : 1;
|
||
// [1] Costruzione lista chiavi
|
||
for (var dt = dtFrom.Date; dt < adesso; dt = dt.AddHours(step))
|
||
{
|
||
// salvo l'elenco datetime...
|
||
dtList.Add(dt);
|
||
|
||
// processo il resto...
|
||
string dateKey = reqGroup switch
|
||
{
|
||
GroupMode.Day => $"{dt:yyyyMMdd}",
|
||
GroupMode.Hour => $"{dt:yyyyMMddHH}",
|
||
_ => $"{dt:yyyyMM}"
|
||
};
|
||
|
||
RedisKey currKey = reqTag switch
|
||
{
|
||
TagMode.Envir => Key($"stats:rtime:env:{dateKey}"),
|
||
TagMode.Mode => Key($"stats:rtime:mode:{dateKey}"),
|
||
_ => ""
|
||
};
|
||
|
||
if (!string.IsNullOrEmpty($"{currKey}"))
|
||
{
|
||
keys.Add(currKey);
|
||
tasks.Add(_db.HashGetAllAsync(currKey));
|
||
}
|
||
}
|
||
|
||
// [2] Lettura in batch (pipeline)
|
||
var allResults = await Task.WhenAll(tasks);
|
||
|
||
// [3] Elaborazione risultati
|
||
var results = new List<RealtimeProcDto>();
|
||
|
||
for (int i = 0; i < allResults.Length; i++)
|
||
{
|
||
var entries = allResults[i];
|
||
if (entries.Length == 0) continue;
|
||
|
||
var dtos = entries
|
||
.GroupBy(e => e.Name.ToString().Split(':')[0])
|
||
.Select(g => new RealtimeProcDto
|
||
{
|
||
Hour = dtList[i],
|
||
TagClass = fixTag(g.Key),
|
||
//TagClass = g.Key,
|
||
Elapsed = (double)g.First(x => x.Name.ToString().EndsWith(":sum")).Value,
|
||
EventCount = (int)g.First(x => x.Name.ToString().EndsWith(":count")).Value
|
||
});
|
||
|
||
results.AddRange(dtos);
|
||
}
|
||
|
||
// calcolo i distinct tag...
|
||
var uniqueTags = results
|
||
.Select(x => x.TagClass)
|
||
.Distinct()
|
||
.OrderBy(x => x)
|
||
.ToList();
|
||
|
||
// ora faccio il fill di zeri...
|
||
for (int i = 0; i < allResults.Length; i++)
|
||
{
|
||
var entries = allResults[i];
|
||
// qui procedo SOLO se NON trovo
|
||
if (entries.Length == 0)
|
||
{
|
||
var dtos = uniqueTags
|
||
.Select(t => new RealtimeProcDto
|
||
{
|
||
Hour = dtList[i],
|
||
TagClass = fixTag(t),
|
||
Elapsed = 0,
|
||
EventCount = 0
|
||
});
|
||
results.AddRange(dtos);
|
||
}
|
||
}
|
||
|
||
return results;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Restituisce elenco statistiche aggregate disponibili dal DB
|
||
/// </summary>
|
||
/// <param name="from"></param>
|
||
/// <param name="to"></param>
|
||
/// <returns></returns>
|
||
public async Task<List<StatsAggregatedModel>> StatsAggrListAsync(DateTime from, DateTime to)
|
||
{
|
||
List<StatsAggregatedModel> answDto = new List<StatsAggregatedModel>();
|
||
answDto = await dbController.StatsAggrGetAsync(from, to);
|
||
return answDto;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Restituisce il periodo valido x dati aggregati
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
public async Task<Utils.DtUtils.Periodo> StatsAggrRangeAsync()
|
||
{
|
||
var answDto = await dbController.StatsAggrRangeAsync();
|
||
return answDto;
|
||
}
|
||
|
||
#endregion Public Methods
|
||
|
||
#region Protected Methods
|
||
|
||
/// <summary>
|
||
/// Esegue un fix dei tags: se fossero numerici li converte in Egw.Window.Data.Enums.QuestionModes
|
||
/// </summary>
|
||
/// <param name="origValue"></param>
|
||
/// <returns></returns>
|
||
protected string fixTag(object origValue)
|
||
{
|
||
string strVal = $"{origValue}";
|
||
// 1. Proviamo a vedere se il valore è un intero
|
||
if (int.TryParse(strVal, out int numericValue))
|
||
{
|
||
// 2. Verifichiamo se il numero è definito nel tuo Enum
|
||
if (Enum.IsDefined(typeof(QuestionModes), numericValue))
|
||
{
|
||
// Trasformiamo il numero nel nome testuale dell'enum
|
||
strVal = ((QuestionModes)numericValue).ToString();
|
||
}
|
||
}
|
||
return strVal;
|
||
}
|
||
|
||
#endregion Protected Methods
|
||
|
||
#region Private Fields
|
||
|
||
private static Logger Log = LogManager.GetCurrentClassLogger();
|
||
|
||
private readonly TimeSpan _archivePeriod;
|
||
|
||
private readonly string _base;
|
||
|
||
private readonly IDatabase _db;
|
||
|
||
private readonly TimeSpan _retention;
|
||
|
||
private readonly Random _rnd = new Random();
|
||
|
||
#endregion Private Fields
|
||
|
||
#region Private Properties
|
||
|
||
private static LuxController dbController { get; set; } = null!;
|
||
|
||
#endregion Private Properties
|
||
|
||
#region Private Methods
|
||
|
||
/// <summary>
|
||
/// Generatore RUID:
|
||
/// ID incrementale = timestamp ms + random 4 chars HEX
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
private string GenerateRuid()
|
||
{
|
||
long ts = DateTimeOffset.Now.ToUnixTimeMilliseconds();
|
||
string rand = Convert.ToString(_rnd.Next(0x1000, 0xFFFF), 16).ToUpper();
|
||
return $"{ts}-{rand}";
|
||
}
|
||
|
||
private RedisKey GetCombinationsKey() => Key("req:combinations");
|
||
|
||
/// <summary>
|
||
/// Restituisce un oggetto HashDict di Redis come dizionario
|
||
/// </summary>
|
||
/// <param name="reqKey"></param>
|
||
/// <returns></returns>
|
||
private async Task<Dictionary<string, RedisValue>> GetRedisHashDict(RedisKey reqKey)
|
||
{
|
||
Dictionary<string, RedisValue> reqDict = new Dictionary<string, RedisValue>();
|
||
var reqData = await _db.HashGetAllAsync(reqKey);
|
||
if (reqDict != null)
|
||
{
|
||
reqDict = reqData.ToDictionary(x => x.Name.ToString(), x => x.Value);
|
||
}
|
||
return reqDict;
|
||
}
|
||
|
||
#if false
|
||
public async Task AggregateAndSaveStatsAsync(TimeSpan? timeRange = null, bool deleteFromRedis = false)
|
||
{
|
||
var now = DateTime.Now;
|
||
|
||
// Se non è specificato un range, usa il default di 24 ore
|
||
var rangeHours = timeRange?.TotalHours ?? 24;
|
||
|
||
// Calcola l'orario di riferimento (ora intera precedente)
|
||
var referenceTime = now.AddMinutes(-now.Minute).AddSeconds(-now.Second);
|
||
var cutoffTime = referenceTime.AddHours(-rangeHours);
|
||
|
||
// Se non si specifica un range, prendiamo le ultime 24 ore
|
||
if (!timeRange.HasValue)
|
||
{
|
||
cutoffTime = referenceTime.AddHours(-24);
|
||
}
|
||
|
||
var aggregatedData = new List<StatsAggregatedModel>();
|
||
var detailData = new List<StatsDetailModel>();
|
||
|
||
// Per ogni ora nel range da processare
|
||
var currentHour = cutoffTime;
|
||
var processedKeys = new HashSet<string>(); // Per evitare duplicati
|
||
|
||
while (currentHour < referenceTime)
|
||
{
|
||
var hourKey = Key($"stats:agg:{currentHour:yyyyMMddHH}");
|
||
|
||
// Verifica se la chiave esiste
|
||
var exists = await _db.KeyExistsAsync(hourKey);
|
||
if (!exists)
|
||
{
|
||
currentHour = currentHour.AddHours(1);
|
||
continue;
|
||
}
|
||
|
||
var entries = await _db.HashGetAllAsync(hourKey);
|
||
if (entries.Length == 0)
|
||
{
|
||
currentHour = currentHour.AddHours(1);
|
||
continue;
|
||
}
|
||
|
||
// Processa le voci per questa ora
|
||
foreach (var entry in entries)
|
||
{
|
||
var key = entry.Name.ToString();
|
||
var value = (double)entry.Value;
|
||
|
||
// Dividi chiave per ottenere envir e tipo
|
||
var parts = key.Split(':');
|
||
if (parts.Length < 2) continue;
|
||
|
||
string groupKey = parts[0];
|
||
string metric = parts[1];
|
||
|
||
// Se è una chiave di tipo "count", "sum", o "max"
|
||
if (metric == "count")
|
||
{
|
||
var count = (long)value;
|
||
|
||
// Recupera i valori sum e max per questo gruppo
|
||
var sumKey = $"{groupKey}:sum";
|
||
var maxKey = $"{groupKey}:max";
|
||
|
||
var sumValue = await _db.HashGetAsync(hourKey, sumKey);
|
||
var maxValue = await _db.HashGetAsync(hourKey, maxKey);
|
||
|
||
double avg = count > 0 ? (double)sumValue / count : 0;
|
||
double max = maxValue.IsNull ? 0 : (double)maxValue;
|
||
|
||
// Estrai envir e tipo
|
||
var envTipo = groupKey.Split('|');
|
||
if (envTipo.Length >= 2)
|
||
{
|
||
var envir = envTipo[0];
|
||
var tipo = envTipo[1];
|
||
|
||
// Aggiungi ai dati aggregati
|
||
aggregatedData.Add(new StatsAggregatedModel
|
||
{
|
||
GroupType = "envir|tipo",
|
||
GroupValue = $"{envir}|{tipo}",
|
||
Hour = currentHour,
|
||
RequestCount = count,
|
||
AvgDuration = avg,
|
||
MaxDuration = max
|
||
});
|
||
|
||
// Aggiungi ai dati dettagliati
|
||
detailData.Add(new StatsDetailModel
|
||
{
|
||
Environment = envir,
|
||
Type = tipo,
|
||
Hour = currentHour,
|
||
RequestCount = count,
|
||
AvgDuration = avg,
|
||
MaxDuration = max
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
// Se richiesto, elimina la chiave da Redis
|
||
if (deleteFromRedis)
|
||
{
|
||
await _db.KeyDeleteAsync(hourKey);
|
||
}
|
||
|
||
currentHour = currentHour.AddHours(1);
|
||
}
|
||
|
||
// Salva i dati sul DB solo se ci sono dati da salvare
|
||
if (aggregatedData.Count > 0)
|
||
{
|
||
await dbController.StatsAggrUpsertAsync(aggregatedData, true);
|
||
await dbController.StatsDetailModelUpsertAsync(detailData, true);
|
||
|
||
Log.Info($"Salvate {aggregatedData.Count} righe su DB per il periodo da {cutoffTime:yyyy-MM-dd HH:mm} a {referenceTime:yyyy-MM-dd HH:mm}");
|
||
}
|
||
else
|
||
{
|
||
Log.Info("Nessun dato da salvare sul DB");
|
||
}
|
||
}
|
||
#endif
|
||
|
||
private RedisKey GetRequestKey(string ruid) => Key($"request:{ruid}");
|
||
|
||
private RedisKey GetSortedSetKey(string envir, string tipo) => Key($"requests:{envir}:{tipo}");
|
||
|
||
private RedisKey GetUidSetKey(string uid) => Key($"uid:{uid}:requests");
|
||
|
||
private RedisKey Key(string suffix) => (RedisKey)($"{_base}:RUID:{suffix}");
|
||
|
||
#endregion Private Methods
|
||
}
|
||
} |