Fixed stream BDF Excel output to .xlsx

BDF_user_pass.txt

@@ -0,0 +1,5 @@
+PRD_3PTY_BDF
+f4H&&5igkEwP
+
+Doug.Macintosh
+/FIjK5zxrW/:

Embedded/mft_passphr.ppk

@@ -0,0 +1,31 @@
+PuTTY-User-Key-File-3: ssh-rsa
+Encryption: aes256-cbc
+Comment: rsa-key-20260324
+Public-Lines: 6
+AAAAB3NzaC1yc2EAAAADAQABAAABAQCGXiwP6IMbePi6ayHSQUqKLgEKc+gVdumb
+jUqp5J8aPcXEkn0ONegEbJIzTkr5HVclbYBwTwQTTNJrNhiRU1AINxG3L0a+dons
+cjjkTFDaXvuiJWCIeMfTcNHFaxb+bYzGNNeaFyS/JEEd8vKMq7Q3M5mg4yZIXXYB
+otZsGzw17gYFX1pVVA2+xyaTgYXTMLwJzbaqj33yKfRjsFS6JrZcmTc7ee7183Y4
+v4QB1HZ2IDJK5IAQEYgEo1QeqowOJuwaCw9DmS/Qs+k9qZBRhxFf6Zq5PuLFXq1N
+uDbzt7sJaEcRmfxZaLG79VgI+POa4lkKPC4jUWBUGlI4BcPrYK4F
+Key-Derivation: Argon2id
+Argon2-Memory: 8192
+Argon2-Passes: 21
+Argon2-Parallelism: 1
+Argon2-Salt: c941ab850062e198224874be51e51de8
+Private-Lines: 14
+ULUmga9nK5WK030clbRbo9LtY3RcLUBTzM09mPzKt8G0j2N7ITVlzoni5nwMU036
+Z0I26GXJudjlq6hE86fjbAbZ6j3OzvYY3guk/5d/Gc+2k/eqg6agUcJcm1rOkGiS
+bwjAIApZvAy1oPU1JvI+MOiWYWIUQzqexLt4ipnpAgA4njlWw+33mLDp+o6SLSyB
+55ckJlC2D6q/oJGV4DYatwcw/AE20lB7DHc5u9eM9ekkRl0xfha5naUnAvqLPyoP
+yxdW/YW3TbBWFwFV1MF7RuongH89m7vvxK1YcQcD7wre8cfW5E9BRdoL34b3T5OB
+nb/yDnFQoVYibuai6Y1dmZjHp/h2aB0L9xBNb3wXfYLVjEVAPBk/xDcxeyWKBfnV
+skHiGIRVkTGI93jmpG39Gwhq29VhYxss/cjTdXwDcEoG2PK4oUSUPjpB0YfjAn3A
+yC7YVME2qZ3u1a5LR7rvoHN3xEPInRDFUY0X47FRo11s3vJFjQ34SF0DO9Nx3oCY
+Cc/6mIEy3xJSehqQl4rnL5N6+JEAjimczt/FS1wwVKFSmnhpKZoz4GhjrX42R3Zn
+3SZeSVa+Sb6gOLibux35irD5Cs7hDOeqXZYmB9/ktfsmIEKr2Q6dXMhVNBfjZoBQ
+9nvcpjQZ7tNlRsR16HBh1HbpxQ4jkOk43fBlKbL0Chq4xHwmMh1KfIKiKUBl5pXs
+kwHGD6QGfy6LhDnTj2cxXCfeVxbJyWxhciUaVvxgiqacxd5gPHszM2WvUqgCPSDG
+Hx5VBeN/XYMFcP9Q+5N6BExm+Q/QKn0p8XelVZm96HHCmmpA9TMzoAt5mm/1MLbQ
+wdLDKScpYHusxZrZEppIOz1ml6Sasid+BoUgdA9fof7pUqAFuMMoiyByGeJe7xSB
+Private-MAC: 1c037a36176788b5153c23baadf2c083eeecbe54cc200a8af404ca2cad1af2cf

Embedded/private_nopassphr.ppk

@@ -1,26 +0,0 @@
-PuTTY-User-Key-File-3: ssh-rsa
-Encryption: none
-Comment: rsa-key-20251222
-Public-Lines: 6
-AAAAB3NzaC1yc2EAAAADAQABAAABAQCkhl4EpiCvmTBENERzih8VHL4AzDJfm08e
-mA+PYg+mWhrM8eZ79zuAdy6NJHlyP0ccdycujqpMAV8EBRZGxqSf74sBQhg9+tzV
-0k2oALfV0CznOX3Fn1QEl3d3f3eNe5O1e7VipKBTGz9CGfTE2MIS3cnYU2g3LtoD
-55sKjXXnUIPRy7YCCK8Rsd/bZITJMX6A5FhBpyuMl0szTf5XI4p/hBW/S9lOiGBc
-AATL+jctjXz7bj6PjCvEBlINXct5h3u7xoL6wNxx9lRayEWRaManYczhXsy9ZaFU
-2TUQYFSN5VTJVIQ8X2Uaip+uPimikdrXImOA51FAUri5esJceuVB
-Private-Lines: 14
-AAABAEW3doGrz+/5Dv3n1BXNsqwkmNMHtFTVICrLrtRIbm9EgVTVMKDZPAqM9lny
-2c+yxrRmPWE6LSm17whqC22EYWAwhovK8TDaa9fjnOqTG1NsOorkzsn+YQDtGj+1
-8PgwJIBj4pHhRtrQkfa1vwXnAB40g4K6nU+8979t1kIbfZm905+esCE2i384U4HZ
-EdxxqJjiT/qSyWJzCocqdEc7/u8wQbbBJa2ES9i/ABh94o0/Yn2Ub8hknIYKCte2
-zzlGlLHQ1r5JxkXEWQhEanv4YX7cmr3KIpz5KZLWm1M/0iD83Ih9mUe8pfLYBMWM
-Ey04F+vtXOKkmWho3+9BipepJREAAACBAM1w11la3UhGujE/bxDwdawUPiC5lZaN
-4MdsdQ4aAsEMFabRpdR8ZIL4QEIZw73+NEAOiBcc7OpiylMO5Areya4lZ2FPE39e
-T0Z/fyqlvYGzSYg4L8f10r56uhwbhsSJRGdrRJfgukHNVjqi14MCN78Ujn/DbjdD
-JYpJleAKaS1NAAAAgQDNA772ganZ1rjvmwEGLCBbFtQGDuf6/iMyuAGWObb8zdea
-76ZwWtZ3DFotSD/K6ajBzCzH7ghJ9L1URjl0pTJb4Gq5Eel61maaG0ijJ02ZMmQL
-vme8Hs7h95J2XcVHaF7DVpNlY/hFXiEyVsVTKWD3LDKhLNM1nbD2WiAafqatxQAA
-AIEAp/ot6RXzIHO9o8dfXibt/HPgz3Ulq9jLEUiMHT7oamXyvqDfdFC4QBwxGwnX
-J7bzDV2FdwLv3Jsv+s90vnQiHf1/wzW9msT7wyyp1FCV8kQaVwcGEC6Ie1ihh3vN
-gZ86hvEdbYMvFIRCheMjoTAJ9BywKQjlq2xaJ8wNGYyunKA=
-Private-MAC: 8fb0374e7e484491cc9a5bc9b649605a7f9da3a8e07580b0c6b60dd6464569f6

Embedded/public_Rogers1!

@@ -1,9 +0,0 @@
----- BEGIN SSH2 PUBLIC KEY ----
-Comment: "rsa-key-20251222"
-AAAAB3NzaC1yc2EAAAADAQABAAABAQCkhl4EpiCvmTBENERzih8VHL4AzDJfm08e
-mA+PYg+mWhrM8eZ79zuAdy6NJHlyP0ccdycujqpMAV8EBRZGxqSf74sBQhg9+tzV
-0k2oALfV0CznOX3Fn1QEl3d3f3eNe5O1e7VipKBTGz9CGfTE2MIS3cnYU2g3LtoD
-55sKjXXnUIPRy7YCCK8Rsd/bZITJMX6A5FhBpyuMl0szTf5XI4p/hBW/S9lOiGBc
-AATL+jctjXz7bj6PjCvEBlINXct5h3u7xoL6wNxx9lRayEWRaManYczhXsy9ZaFU
-2TUQYFSN5VTJVIQ8X2Uaip+uPimikdrXImOA51FAUri5esJceuVB
----- END SSH2 PUBLIC KEY ----

SFTP.cs

@@ -12,6 +12,25 @@ namespace bdf
 {
 	public class sftp
     {
+        public static ConnectionDetails sshPRODDetails = new ConnectionDetails
+        {
+            host = "mft.rogers.com",
+            ppkFile = "BDF.Embedded.mft_passphr.ppk",       //Add "BDF.Embedded." to filename if embedded
+            passPhrase = "Rogers1!",
+            username = "PRD_3PTY_BDF",
+            //password = "f4H&&5igkEwP",
+            port = 50022
+        };
+
+        public static ConnectionDetails sshDEVDetails = new ConnectionDetails
+        {
+            host = "dev.mft.rogers.com",
+            ppkFile = "BDF.Embedded.private_passphr.ppk",       //Add "BDF.Embedded." to filename if embedded
+            passPhrase = "Rogers1!",
+            username = "DEV_APP_BDF",
+            //password = "DEV_APP_BDF"
+        };
+
         public class ConnectionDetails
         {
             public Int32 maxAttempts = 5;
@@ -94,7 +113,7 @@ namespace bdf
                 if (pkFile != null)
                 {
                     auth.Add(new Renci.SshNet.PrivateKeyAuthenticationMethod(cd.username, pkFile)); //preferred
-                    Logger.Log(1, "  -added PPK auth method.");
+                    Logger.Log(2, "  -added PPK auth method.");
                 }
 
             }
@@ -110,7 +129,7 @@ namespace bdf
                 auth.Add(kb);
 
                 auth.Add(new Renci.SshNet.PasswordAuthenticationMethod(cd.username, cd.password));
-                Logger.Log(1, "  -added user/pass auth method.");
+                Logger.Log(2, "  -added user/pass auth method.");
             }
 
             return new Renci.SshNet.ConnectionInfo(cd.host, cd.port, cd.username, auth.ToArray())
@@ -210,7 +229,87 @@ namespace bdf
                 return false;
             }
         }
-        
+
+        public static void Upload_DictStream(Dictionary<string,MemoryStream> Dict, string task, ConnectionDetails cd)
+        { 
+            try
+            {
+                //using (var sftpClient = CreateClient(host, port, username, password))
+                var sftpInfo = BuildConnectionInfo(cd);
+                Renci.SshNet.SftpClient sftpClient = null;
+                try
+                {
+                    sftpClient = TrySftpConnectWithHardTimeout(sftpInfo, cd.maxAttempts, cd.retrySecs);
+                }
+                catch (Renci.SshNet.Common.SshConnectionException e)
+                {
+                    Logger.Log(1, "  connection exception ({0}) : {1}", "", e.Message);
+                }
+
+                if (sftpClient == null)
+                {
+                    Logger.Log(1, "  null connection exception.  Transfer failed.");
+                    return;
+                }
+
+                string cpath = "";
+                foreach (var msFile in Dict)
+                {
+                    string filename = "Outbox" + Path.DirectorySeparatorChar + task + Path.DirectorySeparatorChar + msFile.Key;
+                    MemoryStream ms = msFile.Value;
+
+                    cpath = EnsureSftpDirectoryExists(sftpClient, filename);
+                    // file upload 
+                    try
+                    {
+                        ms.Position = 0;
+                        sftpClient.UploadFile(
+                            ms,
+                            filename,
+                            uploaded =>
+                            {
+                                Logger.Log(2, $"Uploaded {Math.Round((double)uploaded / ms.Length * 100)}% of the file.");
+                            });
+                    }
+                    catch (Renci.SshNet.Common.SshException e)
+                    {
+                        Logger.Log(0, " SSH upload Exception {0}:{1}", e.Message, e.InnerException.Message);
+                    }
+                    Logger.Log(0, "  -upload: {0} ({3}KB) to {1}:{2}", filename, cd.host, cd.port, (ms.Length / 1024) + 1);
+
+                }
+
+                //Console.WriteLine("starting dir pull {0}", cpath);
+                var files = sftpClient.ListDirectory(cpath).Where(f => f.IsRegularFile &&
+                     (f.Name.EndsWith(".pdf", System.StringComparison.OrdinalIgnoreCase) || //pdf
+                     f.Name.EndsWith(".zip", System.StringComparison.OrdinalIgnoreCase) || //pdf
+                     f.Name.EndsWith(".xlsx", System.StringComparison.OrdinalIgnoreCase)));
+
+                Logger.Log(task, " EFT {0} Folder Contents:",task);
+                foreach (var s in files)
+                {
+                    Logger.Log(task, " => {0} ({1}KB) {2}", s.FullName, (s.Attributes.Size/1024)+1, s.Attributes.LastWriteTime);
+                }
+
+                sftpClient.Disconnect();
+                Logger.Log("Disconnected");
+                try
+                {
+                    sftpClient?.Dispose();
+                }
+                catch { }
+
+                return; // success
+
+            }
+            catch (Exception e)
+            {
+                Logger.Log(0, "SFTP exception {0}:{1}", e.Message, e.InnerException.Message);
+            }
+            
+        }
+
+
         public static void Upload_Stream(MemoryStream ms, string filename, ConnectionDetails cd)
         {
             try

Switches.cs

@@ -54,12 +54,25 @@ namespace bdf
                     backBillAmt = amt;
             }
 
+
             if (myargs.Exists("upload"))
             {
                 upload = true;
-                Logger.Log(0, "Files will be securely transferred to EFT");
+                //Logger.Log(0, "Files will be securely transferred to EFT");
+
+                if (myargs.Exists("dev"))
+                {
+                    sshDetails = sftp.sshDEVDetails;
+                    Logger.Log(0, "Warning: Files will be transferred to EFT using DEV server for testing only.");
+                }
+                else
+                {
+                    sshDetails = sftp.sshPRODDetails;
+                    Logger.Log(1, "Files will be securely transferred to EFT using MFT production server.");
+                }
             }
 
+
             if (myargs.Exists("zip"))
             {
                 archive = true;
@@ -226,7 +239,7 @@ namespace bdf
         public static class ExpirationGuard
         {
             // Hardcoded expiration date (UTC recommended)
-            private static readonly DateTime ExpirationDate = new DateTime(2027, 12, 31, 0, 0, 0, DateTimeKind.Utc);
+            private static readonly DateTime ExpirationDate = new DateTime(2026, 12, 31, 0, 0, 0, DateTimeKind.Utc);
 
             public static void ThrowIfExpired()
             {

_Main.cs

@@ -25,21 +25,14 @@ namespace bdf
     class bdf
     {
         public static string product = "BDF";
-        public static string version = "3.5.2";
+        public static string version = "4.0.1";
 
         public static string fullUser = WindowsIdentity.GetCurrent().Name;      // \\DOMAIN\User.Id for NTLM auth for Megatool
 
         //EFT Upload Credentials
         public static bool upload = false;
-        public static sftp.ConnectionDetails sshDetails = new sftp.ConnectionDetails
+        public static sftp.ConnectionDetails sshDetails;        // mapped to either DEV or PROD servers
-        {
+        
-            host = "dev.mft.rogers.com",
-            ppkFile = "BDF.Embedded.private_passphr.ppk",       //Add "BDR.Embedded." to filename if embedded
-            passPhrase = "Rogers1!",
-            username = "DEV_APP_BDF",
-            password = "DEV_APP_BDF"
-        };
-
         public static string S4LT = "Test1234";
 
         public static Dictionary<string, int> BAN = new Dictionary<string, int> { {"LDF", 857412 }, {"WAV", 864507 }, {"SIP", 858701 } };
@@ -59,7 +52,7 @@ namespace bdf
 
         public static bool reportappend = false;
         public static string reportname = "bdf_report.txt";
-        public static StreamWriter reportfile = new StreamWriter(@"" + reportname, reportappend);
+        public static StreamWriter reportfile = new StreamWriter(new FileStream(@"" + reportname, FileMode.Create, FileAccess.Write, FileShare.Read));
 
         public static string domain = "RCI";
         public static string username = "";
@@ -161,6 +154,7 @@ namespace bdf
 
             //outfile = new StreamWriter(@"" + logname, logappend); // defined earlier so logging works in .dll loading
             outfile.AutoFlush = true;
+            reportfile.AutoFlush = true;
 
             //Suppress system error text messages?
             //Console.SetError(TextWriter.Null);
@@ -178,6 +172,7 @@ namespace bdf
             Logger.Log("--- call (613)697-9178 for support ---");
 
             Logger.Log(product, "INIT: Tool version {0} initiated {1}", version, DateTime.Now);
+            Logger.Log(product, "USER: Operated by: {0}", fullUser);
 
             // ***MAIN -------------------------------------------------------------------------------------------------------------------------------
 
@@ -1196,6 +1191,8 @@ namespace bdf
                 }
 
                 //Now it is safe to dump out BDF files as we modified the custom invoice on tab2
+                Dictionary<string, MemoryStream> bdfDossier = new Dictionary<string, MemoryStream> { };
+                MemoryStream bdf_as_ms = new MemoryStream();    //keep a copy of the BDF in case we need to add to an archive MAR26
 
                 {
                     // Output BDF to ./ if not uploading
@@ -1210,11 +1207,6 @@ namespace bdf
 
                         Logger.Log(task, "SUMM> {0} Detailed bill lines added. Monthly totals MRR={1:N2} NRC={2:N2} before taxes", (bdf2.LastDataRow - 1), bdf1[2, 6].Value, bdf1[2, 7].Value);
 
-                        //string fileTraceable = output_path + bdf.filename.Split('.')[0] + traceable + ".xlsx";
-                        if (!FileIsWritable(svc_path + fileTraceable)) return;
-                        //clear older versions of same month
-                        DeleteFilesByPrefix(svc_path + fileTraceable, 7);       // use length of 12 to only clear same billing period files.
-                        
                         ExcelDocumentProperties edp = new ExcelDocumentProperties { };
                         edp.Title = "BDF " + task + " Detail";
                         edp.Subject = task + " Billing Detail File";
@@ -1223,8 +1215,20 @@ namespace bdf
 
                         edp.ApplyTo(workbook);
 
-                        workbook.SaveToFile(svc_path + fileTraceable, ExcelVersion.Version2016);
+                        //Store output docs as memorystreams and deal with them all at once
-                        Logger.Log(0, "Excel file populated! ({0})", svc_path + fileTraceable);
+                        try
+                        {
+                            workbook.SaveToStream(bdf_as_ms, FileFormat.Version2016);
+                        }
+                        catch (Exception e)
+                        {
+                            Logger.Log("  BDF failed to streamify. ({0})", e.Message);
+                            return;
+                        }
+                        bdfDossier.Add(fileTraceable.Substring(upload ? 4 : 2), bdf_as_ms);      // remove TSK\ from start
+                        
+                        //Logger.Log(0, "Excel file populated! ({0})", svc_path + fileTraceable);
+                        Logger.Log(0, "Excel file populated! ({0})", fileTraceable.Substring(upload ? 4 : 2));
                     }
                     catch (Exception e)
                     {
@@ -1233,42 +1237,6 @@ namespace bdf
                         Logger.Log(0, "Please close the workbook ({0}) if you currently have it open in Excel and then retry.", traceable);
                     }
 
-                    // are we uploading to EFT SFTP server?
-                    if (upload)
-                    {
-                        Logger.Log(0, " Initiating {0}-BDF upload to EFT server...", task);
-                        try
-                        {
-                            using (var ms = new MemoryStream())
-                            {
-                                try
-                                {
-                                    workbook.SaveToStream(ms);
-                                }
-                                catch (Exception e)
-                                {
-                                    Logger.Log("  BDF failed to format for SFTP. ({0})", e.Message);
-                                    return;
-                                }
-
-                                try
-                                {
-                                    sftp.Upload_Stream(ms, "Outbox" + Path.DirectorySeparatorChar + fileTraceable, sshDetails);
-                                }
-                                catch (Exception e)
-                                {
-                                    Logger.Log(0, "BDF SFTP Upload Exception: {0}:{1}", e.Message, e.InnerException.Message);
-                                    return;
-                                }
-                                //Logger.Log("  BDF uploaded.");
-                            }
-                        }
-                        catch (Exception e)
-                        {
-                            Logger.Log(0, "BDF SFTP Upload Exception: {0}:{1}", e.Message, e.InnerException.Message);
-                            return;
-                        }
-                    }
                 }
 
                 // Hold memorystream copies of each PDF invoice created for the ZIP/SAVE and SFTP process(es):  filename, ms
@@ -1365,11 +1333,10 @@ namespace bdf
                     InvoiceCell icArg = SIP ? sip : ic; // pick the correct InvoiceCell to pass to customInvoice
 
                     using (MemoryStream ms = PDF.customInvoiceAsMS_taxes(ref Invoice, ref icArg))  // now pass ic, so we can access ic.TX dictionary 
-                    //using (MemoryStream ms = PDF.customInvoiceAsMS(ref Invoice))
                     {
                         if (SIP)
                             Logger.Log(task, "INFO: PO# {0} Creating INVOICE: MRR={1:0.00} NRC={2:0.00} Total_charges= ${3:0.00}", PO.ElementAt(i).Key, sip.Data_Services, sip.Taxes, sip.Total_Charges);
-                        string PDF_path = "_INV_" + current.ToString() + (SIP ? (String.Format("_{0}", PO.ElementAt(i).Key)) : "") + traceable; //"." + Path.DirectorySeparatorChar;
+                        string PDF_path = task.ToUpper() + "_INV_" + current.ToString() + (SIP ? (String.Format("_{0}", PO.ElementAt(i).Key)) : "") + traceable; //"." + Path.DirectorySeparatorChar;
 
                         ms.Position = 0;
                         try
@@ -1380,17 +1347,17 @@ namespace bdf
                         }
                         catch (Exception e)
                         {
-                            Logger.Log(0, "Error Invoice SFTP upload", e.Message);
+                            Logger.Log(0, "Error Invoice dossier addition", e.Message);
                         }
+                        Logger.Log(0, "PDF file populated! ({0})", PDF_path + ".pdf");
                     }
                     Invoice.Clear();
                 }
 
                 //Now we have a Dossier of all PDF invoices, if we are compressing them, then build zip as memorystream and remove all original PDFs from Dossier
                 Logger.Log(2, "We have {0} invoice PDFs prepared", invoiceDossier.Count);
-                //Setup ZIP archive if desired (bool archive=true)
 
-                
+                //Setup ZIP archive if desired (bool archive=true)
                 if (SIP && archive)
                 {
                     byte[] zip;
@@ -1426,79 +1393,236 @@ namespace bdf
                         }
                         zip = zipMS.ToArray();
                     }
+                    // Now clear the dossier and replace with the single zip archive file
+                    foreach (var ms in invoiceDossier.Values)
+                        ms?.Dispose();
                     invoiceDossier.Clear();
+
+                    // Re-Add the zip to the dossier
                     string fname = "_INV_" + current.ToString() + traceable + ".zip";
                     Logger.Log(2, "Adding {0} to Dossier", fname);
                     invoiceDossier.Add(fname, new MemoryStream(zip));
                 }
+
                 // Now we either have a Dictionary of PDFs, or one with a single ZIP file
                 // If we aren't uploading then we just dump the file locally
-                bool deleted = false;
+
+                // Add in all invoice stream(s) to BDF Dossier
                 foreach (var file in invoiceDossier)
                 {
-                    try
+                    bdfDossier.Add(file.Key, file.Value);
-                    {
+                    Logger.Log(2, " - adding invoice file: {0}", file.Key);
-                        //PDF.customInvoiceSaveFile(task, file.Key, file.Value);
+                }
+                
+                // Now we save or upload all files generated
+                List<string> cleared_files = new List<string> { };      // tracks which file prefix have been deleted
 
-                        file.Value.Position = 0;
+                if (!upload)
+                {
+                    foreach (var file in bdfDossier)
+                    {
+                        Logger.Log(2, " + processing file: {0}", file.Key);
                         string srv_path = bdf.upload ? bdf.base_path + "BDF_Archive" + Path.DirectorySeparatorChar : "";
-                        string file_name = (bdf.upload ? task.ToUpper() : ".") + Path.DirectorySeparatorChar + task.ToUpper() + file.Key;
+                        string file_name = (bdf.upload ? task.ToUpper() : ".") + Path.DirectorySeparatorChar + file.Key;
+                        string prefix = file.Key.Substring(0, 14);
 
-                        if (!FileIsWritable(srv_path + file_name)) return;
-                        // clear older versions
-                        if (!deleted)
-                        {
-                            DeleteFilesByPrefix(srv_path + file_name, 7);       // use length of 12 to only clear same billing period files.
-                            deleted = true;
-                        }   
-                        File.WriteAllBytes(srv_path + file_name, file.Value.ToArray());
-                        Logger.Log(0, "Invoice written! ({0})", srv_path + file_name);
-                    }
-                    catch (Exception e)
-                    {
-                        Logger.Log(0, "Error Saving Invoice(s) to File", e.Message);
-                    }
-
-                    if (upload)
                         try
                         {
-                            file.Value.Position = 0;
+                            if (!FileIsWritable(srv_path + file_name)) return;
-                            string file_name = task.ToUpper() + Path.DirectorySeparatorChar + task.ToUpper() + file.Key;
+                            if (!cleared_files.Contains(prefix))
                             {
-                                Logger.Log(0, " Initiating {0}-INV upload to EFT server...", task);
+                                cleared_files.Add(prefix);
-                                try
+                                Logger.Log(2, " - clearing prefix: {0}", prefix);
-                                {
+                                DeleteFilesByPrefix(srv_path + file_name, 16);      // this should clear any previous files from the same year/month
-                                    try
-                                    {
-                                        sftp.Upload_Stream(file.Value, "Outbox" + Path.DirectorySeparatorChar + file_name, bdf.sshDetails);
-                                    }
-                                    catch (Exception e)
-                                    {
-                                        Logger.Log(0, "Invoice SFTP Upload Exception: {0}:{1}", e.Message, e.InnerException.Message);
-                                        return;
-                                    }
-                                }
-                                catch (Exception e)
-                                {
-                                    Logger.Log(0, "INVSFTP Upload Exception: {0}:{1}", e.Message, e.InnerException.Message);
-                                    return;
-                                }
                             }
                         }
                         catch (Exception e)
                         {
-                            Logger.Log(0, "Error Invoice SFTP upload", e.Message);
+                            Logger.Log(0, "Error Clearing Files: {0}", e.Message);
                         }
-                }
-                // Cleanup stuff
 
-                CXLFile.S_ServiceOrders.RemoveRange(0, CXLFile.S_ServiceOrders.Count);
+                        try
+                        {
+                            file.Value.Position = 0;
+                            File.WriteAllBytes(srv_path + file_name, file.Value.ToArray());
+                            Logger.Log(0, " > file written: ({0})", srv_path + file_name);
+                        }
+                        catch (Exception e)
+                        {
+                            Logger.Log(0, "Error Saving Files: {0}", e.Message);
+                        }
+                    }
+                }
+                else // UPLOAD : we are uploading via SFTP, and creating a ZIP archive on Sharepoint MAR29
+                {
+                    try
+                    {    
+                        Logger.Log(0, "Initiating [{0}] upload to EFT Server...", task);
+                        try
+                        {
+                            sftp.Upload_DictStream(bdfDossier, task.ToUpper(), sshDetails);
+                        }
+                        catch (Exception e)
+                        {
+                            Logger.Log(0, "SFTP Upload Exception: {0}:{1}", e.Message, e.InnerException.Message);
+                            return;
+                        }
+                        
+                    }
+                    catch (Exception e)
+                    {
+                        Logger.Log(0, "Error SFTP upload {0}:{1}", e.Message, e.InnerException.Message);
+                    }
+
+                    //Setup ZIP file for Archive folder
+                    byte[] zip;
+                    //using (FileStream zipFS = new FileStream("test.zip", FileMode.Create))
+
+                    using (MemoryStream zipMS = new MemoryStream())
+                    {
+                        using (ZipArchive archive = new ZipArchive(zipMS, ZipArchiveMode.Create, leaveOpen: false))
+                        {
+                            foreach (var entry in bdfDossier)
+                            {
+                                string fileName = entry.Key;
+                                MemoryStream fileStream = entry.Value;
+
+                                Logger.Log(2, " adding {0} to ZIP archive", fileName);
+                                // Create a new entry in the zip archive
+                                ZipArchiveEntry zipEntry = archive.CreateEntry(fileName);
+
+                                // Open the entry stream to write data into the zip file
+                                // 'using' ensures the entry stream is disposed after writing.
+                                using (Stream entryStream = zipEntry.Open())
+                                {
+                                    // Ensure the source stream is at the beginning before copying
+                                    if (fileStream.CanSeek && fileStream.Position != 0)
+                                    {
+                                        fileStream.Seek(0, SeekOrigin.Begin);
+                                    }
+
+                                    // Copy the contents from the source MemoryStream to the destination ZipArchiveEntry stream
+                                    fileStream.CopyTo(entryStream);
+                                }
+                                //invoiceDossier.Remove(entry.Key);
+                            }
+
+                            //Add BDF Tool Run User report to archive
+                            reportfile.Close();
+                            
+                            ZipArchiveEntry report = archive.CreateEntry("Report.txt");
+                            using (Stream reportStream = report.Open())
+                            {
+                                try
+                                {
+                                    using (FileStream fileStream = new FileStream(reportname, FileMode.Open, FileAccess.Read))
+                                    {
+                                        if (fileStream.CanSeek && fileStream.Position != 0)
+                                        {
+                                            //Console.WriteLine("c");
+                                            fileStream.Seek(0, SeekOrigin.Begin);
+                                        }
+                                        //Console.WriteLine("d");
+                                        fileStream.CopyTo(reportStream);
+                                    }
+                                }
+                                catch (FileNotFoundException ex)
+                                {
+                                    Console.Error.WriteLine($"File not found: {ex.FileName}");
+                                }
+                                catch (UnauthorizedAccessException ex)
+                                {
+                                    Console.Error.WriteLine("Access denied to file.{0}", ex.Message);
+                                }
+                                catch (IOException ex)
+                                {
+                                    Console.Error.WriteLine($"I/O error: {ex.Message}");
+                                }
+                                catch (Exception ex)
+                                {
+                                    // fallback (optional, depending on your policy)
+                                    Console.Error.WriteLine($"Unexpected error: {ex.Message}");
+                                }
+                            }
+                            //Re-enable reportfile for rest of run, this time as append.
+                            reportfile = new StreamWriter(new FileStream(@"" + reportname, FileMode.Append, FileAccess.Write, FileShare.Read));
+                            reportfile.AutoFlush = true;
+                        }
+                        zip = zipMS.ToArray();
+                    }
+
+                    // Now write out the single archive file
+                    string srv_path = bdf.base_path + "BDF_Archive" + Path.DirectorySeparatorChar + task.ToUpper() + Path.DirectorySeparatorChar;
+                    string fname = task.ToUpper() + "_Archive_" + current.ToString() + traceable + ".zip";
+                    Logger.Log(0, "Adding {0} to Archive Folder", fname);
+
+                    try
+                    {
+                        if (!FileIsWritable(srv_path + fname)) return;
+                        //Logger.Log(0, "..check for deletion {0}/{1}", srv_path + fname, (srv_path + fname).Length);
+                        DeleteFilesByPrefix(srv_path + fname, (srv_path + fname).Length);      // this should clear any previous files from the same user on the same day
+                    }
+                    catch (Exception e)
+                    {
+                        Logger.Log(0, "Error Clearing Archive Files: {0}", e.Message);
+                    }
+
+                    try
+                    {
+                        File.WriteAllBytes(srv_path + fname, zip);
+                        //Logger.Log(0, " > archive written: ({0})", srv_path + fname);
+                        Logger.Log(0, " > archive written: ({0})", fname);
+                    }
+                    catch (Exception e)
+                    {
+                        Logger.Log(0, "Error Saving Archive Files: {0}", e.Message);
+                    }
+
+                }
+
+                // Cleanup stuff
+                try
+                {
+                    foreach (var ms in invoiceDossier.Values)
+                        ms?.Dispose();
+                    invoiceDossier.Clear();
+                    cleared_files.Clear();
+                }
+                catch (Exception e)
+                {
+                    Logger.Log(0, "Error Disposing 1: {0}", e.Message);
+                }
+
+                try
+                {
+                    foreach (var ms in bdfDossier.Values)
+                        ms?.Dispose();
+                    bdfDossier.Clear();
+                }
+                catch (Exception e)
+                {
+                Logger.Log(0, "Error Disposing 2: {0}", e.Message);
+                }
+
+                try
+                {
+                    foreach (var wb in CXLFile.S_ServiceOrders)
+                        wb?.Dispose();
+                    //CXLFile.S_ServiceOrders.RemoveRange(0, CXLFile.S_ServiceOrders.Count);
+                    CXLFile.S_ServiceOrders.Clear();
+                }
+                catch (Exception e)
+                {
+                    Logger.Log(0, "Error Disposing 3: {0}", e.Message);
+                }
+                
+
             }   // end of MAIN()
 
             Logger.Log("All done!  Have a Great Day :) ");
             Logger.Log(product, "TERM: Successful Completion at {0}", DateTime.Now);
             outfile.Close();
-            reportfile.Close();
+            //reportfile.Close();
 
         } // end of to-do loop
 
@@ -2015,6 +2139,15 @@ namespace bdf
                 Logger.Log(0, $"Error: {ex.Message}");
             }
         }
+
+        public static void DisposeAndClear<T>(ICollection<T> collection) where T : IDisposable
+        {
+            foreach (var item in collection)
+            {
+                item.Dispose();
+            }
+            collection.Clear();
+        }
     }
 
     public class ExcelDocumentProperties

bdf.csproj

@@ -131,7 +131,7 @@
     <EmbeddedResource Include="Embedded\System.ValueTuple.dll" />
     <EmbeddedResource Include="Embedded\System.Numerics.Vectors.dll" />
     <EmbeddedResource Include="Embedded\private_passphr.ppk" />
-    <EmbeddedResource Include="Embedded\private_nopassphr.ppk" />
+    <EmbeddedResource Include="Embedded\mft_passphr.ppk" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
 </Project>

mft_passphr.ppk

@@ -0,0 +1,31 @@
+PuTTY-User-Key-File-3: ssh-rsa
+Encryption: aes256-cbc
+Comment: rsa-key-20260324
+Public-Lines: 6
+AAAAB3NzaC1yc2EAAAADAQABAAABAQCGXiwP6IMbePi6ayHSQUqKLgEKc+gVdumb
+jUqp5J8aPcXEkn0ONegEbJIzTkr5HVclbYBwTwQTTNJrNhiRU1AINxG3L0a+dons
+cjjkTFDaXvuiJWCIeMfTcNHFaxb+bYzGNNeaFyS/JEEd8vKMq7Q3M5mg4yZIXXYB
+otZsGzw17gYFX1pVVA2+xyaTgYXTMLwJzbaqj33yKfRjsFS6JrZcmTc7ee7183Y4
+v4QB1HZ2IDJK5IAQEYgEo1QeqowOJuwaCw9DmS/Qs+k9qZBRhxFf6Zq5PuLFXq1N
+uDbzt7sJaEcRmfxZaLG79VgI+POa4lkKPC4jUWBUGlI4BcPrYK4F
+Key-Derivation: Argon2id
+Argon2-Memory: 8192
+Argon2-Passes: 21
+Argon2-Parallelism: 1
+Argon2-Salt: c941ab850062e198224874be51e51de8
+Private-Lines: 14
+ULUmga9nK5WK030clbRbo9LtY3RcLUBTzM09mPzKt8G0j2N7ITVlzoni5nwMU036
+Z0I26GXJudjlq6hE86fjbAbZ6j3OzvYY3guk/5d/Gc+2k/eqg6agUcJcm1rOkGiS
+bwjAIApZvAy1oPU1JvI+MOiWYWIUQzqexLt4ipnpAgA4njlWw+33mLDp+o6SLSyB
+55ckJlC2D6q/oJGV4DYatwcw/AE20lB7DHc5u9eM9ekkRl0xfha5naUnAvqLPyoP
+yxdW/YW3TbBWFwFV1MF7RuongH89m7vvxK1YcQcD7wre8cfW5E9BRdoL34b3T5OB
+nb/yDnFQoVYibuai6Y1dmZjHp/h2aB0L9xBNb3wXfYLVjEVAPBk/xDcxeyWKBfnV
+skHiGIRVkTGI93jmpG39Gwhq29VhYxss/cjTdXwDcEoG2PK4oUSUPjpB0YfjAn3A
+yC7YVME2qZ3u1a5LR7rvoHN3xEPInRDFUY0X47FRo11s3vJFjQ34SF0DO9Nx3oCY
+Cc/6mIEy3xJSehqQl4rnL5N6+JEAjimczt/FS1wwVKFSmnhpKZoz4GhjrX42R3Zn
+3SZeSVa+Sb6gOLibux35irD5Cs7hDOeqXZYmB9/ktfsmIEKr2Q6dXMhVNBfjZoBQ
+9nvcpjQZ7tNlRsR16HBh1HbpxQ4jkOk43fBlKbL0Chq4xHwmMh1KfIKiKUBl5pXs
+kwHGD6QGfy6LhDnTj2cxXCfeVxbJyWxhciUaVvxgiqacxd5gPHszM2WvUqgCPSDG
+Hx5VBeN/XYMFcP9Q+5N6BExm+Q/QKn0p8XelVZm96HHCmmpA9TMzoAt5mm/1MLbQ
+wdLDKScpYHusxZrZEppIOz1ml6Sasid+BoUgdA9fof7pUqAFuMMoiyByGeJe7xSB
+Private-MAC: 1c037a36176788b5153c23baadf2c083eeecbe54cc200a8af404ca2cad1af2cf

mft_pub_nopassphr.ppk

@@ -0,0 +1,9 @@
+---- BEGIN SSH2 PUBLIC KEY ----
+Comment: "rsa-key-20260324"
+AAAAB3NzaC1yc2EAAAADAQABAAABAQCGXiwP6IMbePi6ayHSQUqKLgEKc+gVdumb
+jUqp5J8aPcXEkn0ONegEbJIzTkr5HVclbYBwTwQTTNJrNhiRU1AINxG3L0a+dons
+cjjkTFDaXvuiJWCIeMfTcNHFaxb+bYzGNNeaFyS/JEEd8vKMq7Q3M5mg4yZIXXYB
+otZsGzw17gYFX1pVVA2+xyaTgYXTMLwJzbaqj33yKfRjsFS6JrZcmTc7ee7183Y4
+v4QB1HZ2IDJK5IAQEYgEo1QeqowOJuwaCw9DmS/Qs+k9qZBRhxFf6Zq5PuLFXq1N
+uDbzt7sJaEcRmfxZaLG79VgI+POa4lkKPC4jUWBUGlI4BcPrYK4F
+---- END SSH2 PUBLIC KEY ----