From 1b49a4a291a3d541b27e893a135b29c1a8944f93 Mon Sep 17 00:00:00 2001 From: "Samuele E. Locatelli" Date: Thu, 17 Oct 2019 16:47:40 +0200 Subject: [PATCH] Update gestione upload... --- INTERFACE/WebLCP/BaseUserControl.cs | 9 +- INTERFACE/WebLCP/JobsList.aspx | 2 + INTERFACE/WebLCP/JobsList.aspx.cs | 78 ++++- INTERFACE/WebLCP/JobsList.aspx.designer.cs | 18 ++ INTERFACE/WebLCP/WIP.aspx | 5 +- INTERFACE/WebLCP/WUC/mod_startJob.ascx | 26 +- INTERFACE/WebLCP/WUC/mod_startJob.ascx.cs | 291 ++++++++++++++---- .../WebLCP/WUC/mod_startJob.ascx.designer.cs | 9 + Jenkinsfile | 2 +- 9 files changed, 355 insertions(+), 85 deletions(-) diff --git a/INTERFACE/WebLCP/BaseUserControl.cs b/INTERFACE/WebLCP/BaseUserControl.cs index 4924b68..3d1816a 100644 --- a/INTERFACE/WebLCP/BaseUserControl.cs +++ b/INTERFACE/WebLCP/BaseUserControl.cs @@ -1,11 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Web; - -namespace WebLCP +namespace WebLCP { - public class BaseUserControl + public class BaseUserControl : SteamWare.UserControl { } } \ No newline at end of file diff --git a/INTERFACE/WebLCP/JobsList.aspx b/INTERFACE/WebLCP/JobsList.aspx index 1c34ba4..35b23c3 100644 --- a/INTERFACE/WebLCP/JobsList.aspx +++ b/INTERFACE/WebLCP/JobsList.aspx @@ -13,6 +13,8 @@
+ +
diff --git a/INTERFACE/WebLCP/JobsList.aspx.cs b/INTERFACE/WebLCP/JobsList.aspx.cs index 90fd9a5..987bd3b 100644 --- a/INTERFACE/WebLCP/JobsList.aspx.cs +++ b/INTERFACE/WebLCP/JobsList.aspx.cs @@ -1,6 +1,82 @@ -namespace WebLCP +using SteamWare; +using System; +using System.Collections.Generic; +using System.Text; + +namespace WebLCP { public partial class JobsList : BaseUserPage { + protected override void Page_Load(object sender, EventArgs e) + { + base.Page_Load(sender, e); + + if (!Page.IsPostBack) + { + lblOutError.Visible = false; + lblOutSuccess.Visible = false; + } + + mod_startJob.eh_ucev += Mod_startJob_eh_ucev; + } + + private void Mod_startJob_eh_ucev(object sender, EventArgs e) + { + // se trovo evento select/reset mostro/nascondo dettagli... + ucEvent evento = (ucEvent)e; + switch (evento.tipoEvento) + { + case ucEvType.Delete: + break; + case ucEvType.Edit: + break; + case ucEvType.New: + break; + case ucEvType.Reset: + break; + case ucEvType.ReqUpdateParent: + // leggo errori e successi attivi... + Dictionary ctrError = mod_startJob.ctrError; + Dictionary ctrSuccess = mod_startJob.ctrSuccess; + + // se ho errori mostro! + if (ctrError.Count > 0) + { + lblOutError.Visible = true; + // compongo output + StringBuilder sb = new StringBuilder(); + sb.Append("------------------------------------
"); + sb.Append("Error: fileCheck not passed!
"); + sb.Append("------------------------------------
"); + foreach (var item in ctrError) + { + sb.Append($"- {item.Key}: {item.Value}
"); + } + sb.Append("------------------------------------
"); + lblOutError.Text = sb.ToString(); + } + else if(ctrSuccess.Count>0) + { + lblOutSuccess.Visible = true; + // compongo output + StringBuilder sb = new StringBuilder(); + sb.Append("------------------------------------
"); + sb.Append("SUCCESS: fileCheck OK!
"); + sb.Append("------------------------------------
"); + foreach (var item in ctrSuccess) + { + sb.Append($"- {item.Key}: {item.Value}
"); + } + sb.Append("------------------------------------
"); + lblOutSuccess.Text = sb.ToString(); + } + + break; + case ucEvType.Selected: + break; + default: + break; + } + } } } \ No newline at end of file diff --git a/INTERFACE/WebLCP/JobsList.aspx.designer.cs b/INTERFACE/WebLCP/JobsList.aspx.designer.cs index befb814..70c8388 100644 --- a/INTERFACE/WebLCP/JobsList.aspx.designer.cs +++ b/INTERFACE/WebLCP/JobsList.aspx.designer.cs @@ -21,6 +21,24 @@ namespace WebLCP { /// protected global::WebLCP.WUC.mod_startJob mod_startJob; + /// + /// Controllo lblOutError. + /// + /// + /// Campo generato automaticamente. + /// Per la modifica, spostare la dichiarazione di campo dal file di progettazione al file code-behind. + /// + protected global::System.Web.UI.WebControls.Label lblOutError; + + /// + /// Controllo lblOutSuccess. + /// + /// + /// Campo generato automaticamente. + /// Per la modifica, spostare la dichiarazione di campo dal file di progettazione al file code-behind. + /// + protected global::System.Web.UI.WebControls.Label lblOutSuccess; + /// /// Controllo mod_jobsList. /// diff --git a/INTERFACE/WebLCP/WIP.aspx b/INTERFACE/WebLCP/WIP.aspx index dfca143..d817bc3 100644 --- a/INTERFACE/WebLCP/WIP.aspx +++ b/INTERFACE/WebLCP/WIP.aspx @@ -9,12 +9,11 @@

www.admodelling.org

- - + +
we'll be back soon...
- diff --git a/INTERFACE/WebLCP/WUC/mod_startJob.ascx b/INTERFACE/WebLCP/WUC/mod_startJob.ascx index 80f005e..cc9a66e 100644 --- a/INTERFACE/WebLCP/WUC/mod_startJob.ascx +++ b/INTERFACE/WebLCP/WUC/mod_startJob.ascx @@ -6,31 +6,37 @@
- Caricamento Nuovo Job +  Caricamento Nuovo Job 
-
+
- + +
-
+
- - - + + + +
-
+
- + - +
diff --git a/INTERFACE/WebLCP/WUC/mod_startJob.ascx.cs b/INTERFACE/WebLCP/WUC/mod_startJob.ascx.cs index 633e5c3..fcc5a4a 100644 --- a/INTERFACE/WebLCP/WUC/mod_startJob.ascx.cs +++ b/INTERFACE/WebLCP/WUC/mod_startJob.ascx.cs @@ -1,85 +1,250 @@ -using SteamWare; +using Newtonsoft.Json; +using SteamWare; using System; +using System.Collections.Generic; using System.IO; +using System.Text.RegularExpressions; namespace WebLCP.WUC { - public partial class mod_startJob : System.Web.UI.UserControl + public partial class mod_startJob : BaseUserControl { + public Dictionary ctrError + { + get + { + Dictionary answ = new Dictionary(); + string rawData = memLayer.ML.StringSessionObj("ctrError"); + if (!string.IsNullOrEmpty(rawData)) + { + answ = JsonConvert.DeserializeObject>(rawData); + } + return answ; + } + set + { + string jsonData = JsonConvert.SerializeObject(value); + memLayer.ML.setSessionVal("ctrError", jsonData); + } + } + public Dictionary ctrSuccess + { + get + { + Dictionary answ = new Dictionary(); + string rawData = memLayer.ML.StringSessionObj("ctrSuccess"); + if (!string.IsNullOrEmpty(rawData)) + { + answ = JsonConvert.DeserializeObject>(rawData); + } + return answ; + } + set + { + string jsonData = JsonConvert.SerializeObject(value); + memLayer.ML.setSessionVal("ctrSuccess", jsonData); + } + } protected void Page_Load(object sender, EventArgs e) { + if (!Page.IsPostBack) + { + cleanupDir(); + } + } + /// + /// pulisce folder upload da files in IN lascaiti da oltre 15 minuti... + /// + private void cleanupDir() + { + //fileMover.obj.deleteOldest(); } protected void lbtUpload_Click(object sender, EventArgs e) { - // controllo se ho i 2 files... - if (FUplCsv_A.FileName != "" && FUplCsv_B.FileName != "") + checkAndUploadFile(true); + } + + private void checkAndUploadFile(bool doUpload) + { + // init dei dizionari errori/successi + ctrError = new Dictionary(); + ctrSuccess = new Dictionary(); + + // inizio controlli... + if (FUplCsv_A.FileName != "" && (FUplCsv_A.FileName.EndsWith(".txt") || FUplCsv_A.FileName.EndsWith(".csv"))) { - // creo record - DS_App.JobsListDataTable tabJobs = dataLayer.man.taJL.insertNew(txtJobDescr.Text.Trim(), devicesAuthProxy.stObj.utente); - if (tabJobs.Count == 1) + ctrSuccess.Add("DB_INPUT", "OK Filename DB INPUT"); + } + else + { + ctrError.Add("DB_INPUT", "WRONG Filename DB INPUT (only *.csv and *.txt)"); + } + if (FUplCsv_B.FileName != "" && (FUplCsv_B.FileName.EndsWith(".txt") || FUplCsv_B.FileName.EndsWith(".csv"))) + { + ctrSuccess.Add("DB_INPUT_CLU", "OK Filename DB INPUT CLU"); + } + else + { + ctrError.Add("DB_INPUT_CLU", "WRONG Filename DB INPUT CLU (only *.csv and *.txt)"); + } + + // se ho errori mostro...... + if (ctrError.Count == 0) + { + // se solo controllo --> folder temporanea (che poi elimino) + if (!doUpload) { - /************************************************* - * CON il nome JOB (ID) creato... - * - devo creare una folder x il job - * - nella folder devo salvare i 2 files - * - sposto folder in area IN - * - devo creare un file *.req (anche vuoto) della richiesta di calcolo (in area IN) - *************************************************/ - - // recupero riga... - DS_App.JobsListRow currJob = tabJobs[0]; - // uso il nome job creato x le path temporanee con 16 char... - string jobName = "J"+currJob.JobId.ToString("D9"); - // creo upload dir... - string sourceDir = Server.MapPath($"{memLayer.ML.CRS("tempUploadDir")}\\{jobName}\\"); + string jobName = $"TMP{DateTime.Now.ToString("yyyyMMdd_HHmmss_fff")}"; + // carico localmente files + string sourceDir = uploadLocally(jobName); + // controllo i files... + checkFilesUploaded(sourceDir); + // elimino file locali... fileMover.obj.setDirectory(sourceDir); - fileMover.obj.checkDir(); - // proseguo con upload files... - string fileNameA = Path.GetFileName(FUplCsv_A.FileName); - string fileNameB = Path.GetFileName(FUplCsv_B.FileName); - FUplCsv_A.SaveAs($"{sourceDir}{fileNameA}"); - FUplCsv_B.SaveAs($"{sourceDir}{fileNameB}"); - - // sposto folder in dir target.. - - // creo cartella in server target -#if false - //string sourceDir= Server.MapPath($"{tempUploadDir}"); - //string destDir = Server.MapPath($"{memLayer.ML.CRS("clusterBaseDir")}\\IN\\"); // @"L:\IN\"; -#endif - string destDir = $"{memLayer.ML.CRS("clusterBaseDir")}\\IN\\"; // @"L:\IN\"; -#if false - fileMover.obj.copiaFile(sourceDir, destDir, FUplCsv_A.FileName); - fileMover.obj.copiaFile(sourceDir, destDir, FUplCsv_B.FileName); -#endif - fileMover.DirectoryCopy(sourceDir, destDir+$"{jobName}\\", true); - // aggiungo file di request... - File.WriteAllText($"{destDir}\\{jobName}.req", ""); -#if false - //File.Create($"{destDir}\\{jobName}.req"); -#endif - - // NON VA con auth di rete (ha già creato?!?) -#if false - string destDir = $"{memLayer.ML.CRS("clusterBaseDir")}\\IN"; - NetworkCredential destCred = new NetworkCredential + fileMover.obj.eliminaDir(); + } + else + { + // creo record + DS_App.JobsListDataTable tabJobs = dataLayer.man.taJL.insertNew(txtJobDescr.Text.Trim(), devicesAuthProxy.stObj.utente); + if (tabJobs.Count == 1) { - Domain = "", - UserName = "fbfusr", - Password = "fbfpwd2019!" - }; - // trasferisco dati - fileMover.copiaFileL2N(sourceDir, destDir, destCred, FUplCsv_A.FileName, "File_A.csv"); - fileMover.copiaFileL2N(sourceDir, destDir, destCred, FUplCsv_B.FileName, "File_B.csv"); -#endif + /************************************************* + * CON il nome JOB (ID) creato... + * - devo creare una folder x il job + * - nella folder devo salvare i 2 files + * - sposto folder in area IN + * - devo creare un file *.req (anche vuoto) della richiesta di calcolo (in area IN) + *************************************************/ - // registro trasferimento effettuato... - int sizeKb = (FUplCsv_A.PostedFile.ContentLength + FUplCsv_B.PostedFile.ContentLength) / 1024; - // update record! - dataLayer.man.taJL.updateUpload(currJob.JobId, DateTime.Now, sizeKb, destDir); + // recupero riga... + DS_App.JobsListRow currJob = tabJobs[0]; + // uso il nome job creato x le path temporanee con 16 char... + string jobName = "J" + currJob.JobId.ToString("D9"); + + // carico localmente files + string sourceDir = uploadLocally(jobName); + // sposto su cluster + string destDir = moveToCluster(jobName, sourceDir); + // aggiorno su DB + updateDbRecord(currJob, destDir); + } + else + { + ctrError.Add("DB_JOB", "Error creating record on DB"); + } } } + + + // se ho errori mostro...... + if (ctrError.Count > 0) + { + raiseEvent(ucEvType.ReqUpdateParent); + } + } + /// + /// Controllo dei fiels caricati localmente + /// + /// + private void checkFilesUploaded(string sourceDir) + { + string regExpHead = @"^\w*;\w*;\w*;\w*$"; ; + string[] linesDbIn; + + // controllo DB_IN + linesDbIn = File.ReadAllLines($"{sourceDir}DB_INPUT.csv"); + string regExpBody = @"^\d*;(0|1);(pos|neg);\-?\d+\,\d+d*$"; + checkAllFile(linesDbIn, regExpHead, regExpBody); + + // controllo DB_IN_CLU + linesDbIn = File.ReadAllLines($"{sourceDir}DB_INPUT_CLU.csv"); + regExpBody = @"^(pos|neg);\d*;(0|1);\-?\d+\,\d+d*$"; + checkAllFile(linesDbIn, regExpHead, regExpBody); + } + /// + /// controllo il file... + /// + /// + /// + /// + private void checkAllFile(string[] linesDbIn, string regExpHead, string regExpBody) + { + // controllo abbia ALMENO 2 linee (testata + valore) + if (linesDbIn.Length > 1) + { + Regex rgxHead = new Regex(regExpHead); + Regex rgxBody = new Regex(regExpBody); + // controllo linea intestazione + if (!rgxHead.IsMatch(linesDbIn[0])) + { + ctrError.Add("DB_INPUT.csv", $"HEADER ERROR (mismatch): found --> {linesDbIn[0]}"); + } + + // controllo linee successive + for (int i = 1; i < linesDbIn.Length; i++) + { + if (!rgxBody.IsMatch(linesDbIn[i])) + { + ctrError.Add("DB_INPUT.csv", $"DATA ERROR (mismatch): line {i} --> {linesDbIn[i]}"); + } + } + } + else + { + ctrError.Add("DB_INPUT.csv", "Expected almost 2 lines (header + data)"); + } + } + + /// + /// aggiornamento record su DB post upload + /// + /// + /// + private void updateDbRecord(DS_App.JobsListRow currJob, string destDir) + { + // registro trasferimento effettuato... + int sizeKb = (FUplCsv_A.PostedFile.ContentLength + FUplCsv_B.PostedFile.ContentLength) / 1024; + // update record! + dataLayer.man.taJL.updateUpload(currJob.JobId, DateTime.Now, sizeKb, destDir); + } + + /// + /// Sposta i dati da directory locale a cluster + /// + /// + /// + /// + private static string moveToCluster(string jobName, string sourceDir) + { + // creo cartella in server target + string destDir = $"{memLayer.ML.CRS("clusterBaseDir")}\\IN\\"; // @"L:\IN\"; + + // sposto folder in dir target.. + fileMover.DirectoryCopy(sourceDir, destDir + $"{jobName}\\", true); + + // aggiungo file di request... + File.WriteAllText($"{destDir}\\{jobName}.req", ""); + return destDir; + } + /// + /// Carica localmente (sul server) i dati e restituisce path locale (compreso di jobName nella path) + /// + /// + /// + private string uploadLocally(string jobName) + { + // creo upload dir... + string sourceDir = Server.MapPath($"{memLayer.ML.CRS("tempUploadDir")}\\{jobName}\\"); + fileMover.obj.setDirectory(sourceDir); + fileMover.obj.checkDir(); + // proseguo con upload files... + string fileNameA = Path.GetFileName(FUplCsv_A.FileName); + string fileNameB = Path.GetFileName(FUplCsv_B.FileName); + FUplCsv_A.SaveAs($"{sourceDir}DB_INPUT.csv"); + FUplCsv_B.SaveAs($"{sourceDir}DB_INPUT_CLU.csv"); + return sourceDir; } } } \ No newline at end of file diff --git a/INTERFACE/WebLCP/WUC/mod_startJob.ascx.designer.cs b/INTERFACE/WebLCP/WUC/mod_startJob.ascx.designer.cs index d17f8ff..15d77fe 100644 --- a/INTERFACE/WebLCP/WUC/mod_startJob.ascx.designer.cs +++ b/INTERFACE/WebLCP/WUC/mod_startJob.ascx.designer.cs @@ -21,6 +21,15 @@ namespace WebLCP.WUC { /// protected global::System.Web.UI.WebControls.TextBox txtJobDescr; + /// + /// Controllo rfvJobDescr. + /// + /// + /// Campo generato automaticamente. + /// Per la modifica, spostare la dichiarazione di campo dal file di progettazione al file code-behind. + /// + protected global::System.Web.UI.WebControls.RequiredFieldValidator rfvJobDescr; + /// /// Controllo HyperLink1. /// diff --git a/Jenkinsfile b/Jenkinsfile index 8e50849..6e09dc4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,7 @@ pipeline { /* calcolo numero versione... diverso x branch MASTER/DEVELOP */ script { - withEnv(['NEXT_BUILD_NUMBER=42']) { + withEnv(['NEXT_BUILD_NUMBER=43']) { // env.versionNumber = VersionNumber(versionNumberString : '0.8.${BUILD_DATE_FORMATTED, "yyMM"}.${BUILDS_ALL_TIME}', projectStartDate : '2019-07-01', skipFailedBuilds: true) env.versionNumber = VersionNumber(versionNumberString : '0.8.${BUILD_DATE_FORMATTED, "yyMM"}.${BUILDS_ALL_TIME}', projectStartDate : '2019-07-01', skipFailedBuilds: true, overrideBuildsAllTime: '${NEXT_BUILD_NUMBER}') env.APP_NAME = 'LCP'