using Microsoft.Extensions.Caching.Distributed; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using MP.FileData; using Newtonsoft.Json; using NLog; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Diagnostics; using MP.FileData.Controllers; using MP.FileData.DTO; using StackExchange.Redis; using MP.FileData.DbModels; namespace MP.Prog.Data { public class FileArchDataService : IDisposable { #region Public Fields public static FileData.Controllers.FileController dbController; #endregion Public Fields #region Public Constructors public FileArchDataService(IConfiguration configuration, ILogger logger, IConnectionMultiplexer redisConnMult) { _logger = logger; _configuration = configuration; // Conf cache redisConn = redisConnMult; redisDb = this.redisConn.GetDatabase(); // json serializer... FIX errore loop circolare https://www.ryadel.com/en/jsonserializationexception-self-referencing-loop-detected-error-fix-entity-framework-asp-net-core/ JSSettings = new JsonSerializerSettings() { ReferenceLoopHandling = ReferenceLoopHandling.Ignore }; // conf DB string connStr = _configuration.GetConnectionString("MP.Prog"); if (string.IsNullOrEmpty(connStr)) { _logger.LogError("ConnString empty!"); } else { dbController = new FileData.Controllers.FileController(configuration); _logger.LogInformation("DbController OK"); } } #endregion Public Constructors #region Public Methods /// /// Esegue Upsert record ArchivioMacchina /// /// /// public async Task ArchivioMaccUpsert(ArchMaccModel currRec) { bool fatto = await dbController.ArchMaccUpsert(currRec); await ResetArchiveCache(); return fatto; } public async Task> ArchMaccGetAll() { string source = "DB"; List dbResult = new List(); string currKey = $"{redisBaseAddr}:ArchMacc:ALL"; Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); string? rawData = await redisDb.StringGetAsync(currKey); if (!string.IsNullOrEmpty(rawData)) { source = "REDIS"; var tempResult = JsonConvert.DeserializeObject>(rawData); if (tempResult == null) { dbResult = new List(); } else { dbResult = tempResult; } } else { dbResult = dbController.ArchMaccGetAll(); rawData = JsonConvert.SerializeObject(dbResult, JSSettings); await redisDb.StringSetAsync(currKey, rawData, LongCache); } if (dbResult == null) { dbResult = new List(); } stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; Log.Info($"ArchMaccGetAll | {source} in: {ts.TotalMilliseconds} ms"); return dbResult; } public async Task ArchMaccGetByKey(string idxMacchina) { string source = "DB"; ArchMaccModel dbResult = new ArchMaccModel(); string currKey = $"{redisBaseAddr}:ArchMacc:{idxMacchina}"; Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); string? rawData = await redisDb.StringGetAsync(currKey); if (!string.IsNullOrEmpty(rawData)) { source = "REDIS"; var tempResult = JsonConvert.DeserializeObject(rawData); if (tempResult == null) { dbResult = new ArchMaccModel(); } else { dbResult = tempResult; } } else { dbResult = dbController.ArchMaccGetByKey(idxMacchina); rawData = JsonConvert.SerializeObject(dbResult, JSSettings); await redisDb.StringSetAsync(currKey, rawData, LongCache); } if (dbResult == null) { dbResult = new ArchMaccModel(); } stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; Log.Info($"ArchMaccGetByKey | {source} in: {ts.TotalMilliseconds} ms"); return dbResult; } public void Dispose() { // Clear database controller dbController.Dispose(); } public async Task FileCountFilt(SelectData CurrFilter) { int numCount = 0; Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); numCount = dbController.FileCountFilt(CurrFilter.IdxMacchina, CurrFilter.OnlyActive, CurrFilter.OnlyMod, CurrFilter.OnlyNoTag, CurrFilter.FileName, CurrFilter.UserName, CurrFilter.Tag, CurrFilter.SearchVal); stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; Log.Trace($"Effettuata lettura da DB per FileCountFilt: {ts.TotalMilliseconds} ms"); return await Task.FromResult(numCount); } /// /// Recupera un record data chiave /// /// /// public FileModel FileGetByKey(int FileId) { return dbController.FileGetByKey(FileId); } /// /// Recupera tutte le revisioni dato un FileId (ordinate desc) /// /// /// public List FileGetAllRevByKey(int FileId) { return dbController.FileGetAllRevByKey(FileId); } /// /// Restituisce il file dato un ID + revisione /// cerca a pari nome, macchina + REV specifica) /// /// Id del file originale /// Rev specifica richiesta /// public FileModel FileGetByKeyRev(int FileId, int Rev) { return dbController.FileGetByKeyRev(FileId, Rev); } public async Task> FileGetFilt(SelectData CurrFilter) { List dbResult = new List(); Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); dbResult = dbController.FileGetFilt(CurrFilter.IdxMacchina, CurrFilter.OnlyActive, CurrFilter.OnlyMod, CurrFilter.OnlyNoTag, CurrFilter.FileName, CurrFilter.UserName, CurrFilter.Tag, CurrFilter.SearchVal, CurrFilter.NumSkip, CurrFilter.PageSize).ToList(); stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; Log.Trace($"Effettuata lettura da DB per FileGetFilt: {ts.TotalMilliseconds} ms"); return await Task.FromResult(dbResult); } public async Task FlushRedisCache() { RedisValue pattern = new RedisValue($"{redisBaseAddr}:*"); bool answ = await ExecFlushRedisPattern(pattern); } public async Task> GetArchiveStatus() { string source = "DB"; List dbResult = new List(); string currKey = $"{redisBaseAddr}:ArchStatus:ALL"; Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); string? rawData = await redisDb.StringGetAsync(currKey); if (!string.IsNullOrEmpty(rawData)) { source = "REDIS"; var tempResult = JsonConvert.DeserializeObject>(rawData); if (tempResult == null) { dbResult = new List(); } else { dbResult = tempResult; } } else { dbResult = dbController.GetArchiveStatus(); rawData = JsonConvert.SerializeObject(dbResult, JSSettings); await redisDb.StringSetAsync(currKey, rawData, LongCache); } if (dbResult == null) { dbResult = new List(); } stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; Log.Info($"GetArchiveStatus | {source} in: {ts.TotalMilliseconds} ms"); return dbResult; } public async Task> MachineList() { List answ = new List(); answ.Add(new AutocompleteModel { LabelField = "--- TUTTE ---", ValueField = "*" }); answ.AddRange(dbController.ArchMaccGetAll().Select(x => new AutocompleteModel { LabelField = $"{x.IdxMacchina} | {x.Nome} {x.Descrizione} ", ValueField = x.IdxMacchina }).ToList()); return await Task.FromResult(answ); } public void rollBackEdit(object item) { dbController.RollBackEntity(item); } public async Task> TagGetFilt(string SearchVal) { return await Task.FromResult(dbController.TagGetFilt(SearchVal, 200).ToList()); } public Task> TagGetSearch(string searchVal, int numRecord) { List answ = new List(); answ.Add(new AutocompleteModel { LabelField = "--- TUTTE ---", ValueField = "*" }); if (numRecord > -1) { answ.AddRange(dbController.TagGetFilt(searchVal, numRecord).Select(x => new AutocompleteModel { LabelField = $"{x.TagId}", ValueField = x.TagId }).ToList()); } return Task.FromResult(answ); } /// /// Aggiorna intero archivio scansionando dati x tutte le macchine che hanno un path valido /// /// Numero giorni x ricerca all'indietro da data corrente / 0 = nessun limite /// Indica se forzare il tag /// Utente attivo /// indica se vada approvata la modifica o lasciato "in sospeso" /// public async Task UpdateAllArchive(int numDayPre, bool forceTag, string UserName, bool DoApprove) { int checkDone = 0; var listaMacchine = await ArchMaccGetAll(); foreach (var item in listaMacchine.Where(x => !string.IsNullOrEmpty(x.BasePath)).ToList()) { checkDone += await UpdateMachineArchive(item.IdxMacchina, numDayPre, forceTag, false, UserName, DoApprove); } return checkDone; } /// /// Aggiorna archivio di una macchina scansionando path relativo /// /// Codice macchina /// /// Numero giorni x ricerca all'indietro da data corrente / 0 = nessun limite /// /// Forza la riverifica dei tags (x update da setup) /// Scrittura log verboso macchina /// Utente attivo /// indica se vada approvata la modifica o lasciato "in sospeso" /// public async Task UpdateMachineArchive(string idxMacchina, int numDayPre, bool forceTag, bool fullLog, string UserName, bool DoApprove) { int checkDone = 0; Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); string ruleName = "Rule00.json"; try { ArchMaccModel macchina = await ArchMaccGetByKey(idxMacchina); if (macchina != null && !string.IsNullOrEmpty(macchina.BasePath)) { if (!string.IsNullOrEmpty(macchina.RuleName)) { ruleName = macchina.RuleName; // gestione confRule... SearchRules currRule = new SearchRules(); try { string rawData = File.ReadAllText(FileController.rulePath(ruleName)); currRule = JsonConvert.DeserializeObject(rawData); //Log.Info($"Conf rule acquisito da file {ruleName}:{Environment.NewLine}{rawData}"); } catch (Exception exc) { Log.Error($"Eccezione in deserializzazione conf rule{Environment.NewLine}{exc}"); } // se NON deserializzato inizializzo hard-coded if (currRule.Name == "ND") { // fare: lettura conf rule x recupero tag x singola macchina //$"\\b{fileName}" + @".{0,2}\([\w\d\s.]+\)"; Dictionary confReplace = new Dictionary(); confReplace.Add("(", " "); confReplace.Add(")", " "); Dictionary fileExtReplace = new Dictionary(); fileExtReplace.Add(".P-2", ""); // hard coded + salvataggio conf x creare json currRule = new SearchRules() { Name = "Commento Filename", Mode = SearchMode.StringOnFile, MaxChar2Search = 100, ReplaceCR = true, RegExPattern = "\\b{{fileName}}" + @".{0,2}\([\w\d\s./]+\)", RegExRepFileName = true, FileNameExtReplace = fileExtReplace, ExcludedTags = new List() { "M4", "M5", "M4+A", "M4+B", "M5+A", "M5+B" }, OutReplace = confReplace, OutExcludeFileName = true }; if (fullLog) { // serializzo string rawRule = JsonConvert.SerializeObject(currRule, Formatting.Indented); Log.Trace($"Conf rule generato:{Environment.NewLine}{rawRule}"); } } checkDone = dbController.CheckFileArchived(macchina.IdxMacchina, macchina.BasePath, numDayPre, "*.*", forceTag, currRule, UserName, DoApprove); } } } catch (Exception exc) { Log.Error($"Eccezione in UpdateMachineArchive{Environment.NewLine}{exc}{Environment.NewLine}{exc.InnerException}"); } stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; Log.Info($"Effettuato update archivio file MACCHINA | last {numDayPre} days | {checkDone} checked | {ts.TotalMilliseconds} ms"); // svuoto cache! await ResetArchiveCache(); // restituisco conteggio return checkDone; } /// /// Aggiorna archivio di una amcchina scansionando path relativo /// /// Codice macchina /// Elenco dei files (su DB) da verificare /// Forza la riverifica dei tags (x update da setup) /// Scrittura log verboso macchina /// Utente attivo /// indica se vada approvata la modifica o lasciato "in sospeso" /// public async Task UpdateMachineFilesArchive(string idxMacchina, List FileList, bool forceTag, bool fullLog, string UserName, bool DoApprove) { int checkDone = 0; Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); string ruleName = "Rule00.json"; try { ArchMaccModel macchina = await ArchMaccGetByKey(idxMacchina); if (macchina != null && !string.IsNullOrEmpty(macchina.BasePath)) { if (!string.IsNullOrEmpty(macchina.RuleName)) { ruleName = macchina.RuleName; // gestione confRule... SearchRules currRule = new SearchRules(); try { string rawData = File.ReadAllText(FileController.rulePath(ruleName)); currRule = JsonConvert.DeserializeObject(rawData); } catch (Exception exc) { Log.Error($"Eccezione in deserializzazione conf rule{Environment.NewLine}{exc}"); } // se NON deserializzato inizializzo hard-coded if (currRule.Name == "ND") { // fare: lettura conf rule x recupero tag x singola macchina //$"\\b{fileName}" + @".{0,2}\([\w\d\s.]+\)"; Dictionary confReplace = new Dictionary(); confReplace.Add("(", " "); confReplace.Add(")", " "); Dictionary fileExtReplace = new Dictionary(); fileExtReplace.Add(".P-2", ""); // hard coded + salvataggio conf x creare json currRule = new SearchRules() { Name = "Commento Filename", Mode = SearchMode.StringOnFile, MaxChar2Search = 100, ReplaceCR = true, RegExPattern = "\\b{{fileName}}" + @".{0,2}\([\w\d\s./]+\)", RegExRepFileName = true, FileNameExtReplace = fileExtReplace, ExcludedTags = new List() { "M4", "M5", "M4+A", "M4+B", "M5+A", "M5+B" }, OutReplace = confReplace, OutExcludeFileName = true }; if (fullLog) { // serializzo string rawRule = JsonConvert.SerializeObject(currRule, Formatting.Indented); Log.Trace($"Conf rule generato:{Environment.NewLine}{rawRule}"); } } checkDone = dbController.CheckFileListArchived(macchina.IdxMacchina, FileList, macchina.BasePath, "*.*", forceTag, currRule, UserName, DoApprove); } } } catch (Exception exc) { Log.Error($"Eccezione in UpdateMachineArchive{Environment.NewLine}{exc}{Environment.NewLine}{exc.InnerException}"); } stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; Log.Info($"Effettuato update archivio file MACCHINA | # file {FileList.Count} | {checkDone} checked | {ts.TotalMilliseconds} ms"); // svuoto cache! await ResetArchiveCache(); // restituisco conteggio return checkDone; } #endregion Public Methods #region Internal Methods /// /// Approvazione modifica file /// /// /// /// internal async Task FileModApprove(FileModel currItem, string UserName) { bool answ = dbController.FileModApprove(currItem, UserName); // svuoto cache! await ResetArchiveCache(); return answ; } /// /// Forza registrazione della sola approvazione utente + data /// /// /// /// internal async Task FileSetUserApp(FileModel currItem, string UserName) { bool answ = dbController.FileSetUserApp(currItem, UserName); // svuoto cache! await ResetArchiveCache(); return answ; } /// /// Eliminazione file (x rifiuto modifica) /// /// /// internal async Task FileDelete(FileModel currItem) { bool answ = dbController.FileDelete(currItem); // svuoto cache! await ResetArchiveCache(); return answ; } internal async Task FileExport(FileModel currItem) { bool answ = dbController.FileExport(currItem); // svuoto cache! await ResetArchiveCache(); return answ; } internal async Task FileReject(FileModel currItem) { bool answ = dbController.FileModReject(currItem); // svuoto cache! await ResetArchiveCache(); return answ; } internal async Task FileUpdate(FileModel updItem) { bool answ = dbController.FileUpdate(updItem); // svuoto cache! await ResetArchiveCache(); return answ; } internal void ResetController() { dbController.ResetController(); } #endregion Internal Methods #region Protected Fields protected static string connStringBBM = ""; protected static string connStringFatt = ""; #endregion Protected Fields #region Private Fields /// /// gestione key Redis /// private const string redisBaseAddr = "MP:PROG"; private static IConfiguration _configuration; private static ILogger _logger; private static List ElencoMacchine = new List(); private static JsonSerializerSettings? JSSettings; private static NLog.Logger Log = LogManager.GetCurrentClassLogger(); /// /// Durata cache lunga IN SECONDI /// private int cacheTtlLong = 60 * 5; /// /// Durata cache breve IN SECONDI /// private int cacheTtlShort = 60 * 1; /// /// Oggetto per connessione a REDIS /// private IConnectionMultiplexer redisConn; /// /// Oggetto DB redis da impiegare x chiamate R/W /// private IDatabase redisDb = null!; private Random rnd = new Random(); #endregion Private Fields #region Private Properties /// /// Durata cache lunga (+ perturbazione percentuale +/-10%) /// private TimeSpan FastCache { get => TimeSpan.FromSeconds(cacheTtlShort * rnd.Next(900, 1100) / 1000); } /// /// Durata cache lunga (+ perturbazione percentuale +/-10%) /// private TimeSpan LongCache { get => TimeSpan.FromSeconds(cacheTtlLong * rnd.Next(900, 1100) / 1000); } /// /// Durata cache lunga (+ perturbazione percentuale +/-10%) /// private TimeSpan UltraLongCache { get => TimeSpan.FromSeconds(cacheTtlLong * 10 * rnd.Next(900, 1100) / 1000); } #endregion Private Properties #region Private Methods /// /// Esegue flush memoria redis dato pattern /// /// /// private async Task ExecFlushRedisPattern(RedisValue pat2Flush) { bool answ = false; var masterEndpoint = redisConn.GetEndPoints() .Where(ep => redisConn.GetServer(ep).IsConnected && !redisConn.GetServer(ep).IsReplica) .FirstOrDefault(); // sepattern è "*" elimino intero DB... if (masterEndpoint != null && (pat2Flush.Equals(new RedisValue("*")) || pat2Flush == RedisValue.Null)) { redisConn.GetServer(masterEndpoint).FlushDatabase(database: redisDb.Database); } else { var server = redisConn.GetServer(masterEndpoint); var keys = server.Keys(database: redisDb.Database, pattern: pat2Flush, pageSize: 1000); var deleteTasks = new List(); foreach (var key in keys) { deleteTasks.Add(redisDb.KeyDeleteAsync(key)); if (deleteTasks.Count >= 1000) { await Task.WhenAll(deleteTasks); deleteTasks.Clear(); } } if (deleteTasks.Count > 0) { await Task.WhenAll(deleteTasks); } } answ = true; #if false var listEndpoints = redisConn.GetEndPoints(); foreach (var endPoint in listEndpoints) { //var server = redisConnAdmin.GetServer(listEndpoints[0]); var server = redisConn.GetServer(endPoint); if (server != null) { var keyList = server.Keys(redisDb.Database, pattern); foreach (var item in keyList) { await redisDb.KeyDeleteAsync(item); } answ = true; } } #endif return answ; } /// /// Reset della cache dati ArchivioMacchine in redis /// /// private async Task ResetArchiveCache() { await ExecFlushRedisPattern($"{redisBaseAddr}:ArchMacc:*"); await ExecFlushRedisPattern($"{redisBaseAddr}:ArchStatus:*"); } #endregion Private Methods } }