From a067ebf6636e65dc4138a6c47802a095bec8056b Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Mon, 1 Sep 2025 23:47:07 -0500 Subject: [PATCH 1/8] Massive changes to file pushing --- Parallel.Cli/Commands/DecryptCommand.cs | 104 ----------- Parallel.Cli/Commands/EncryptCommand.cs | 109 ----------- Parallel.Cli/Commands/HistoryCommand.cs | 14 +- Parallel.Cli/Commands/PushCommand.cs | 6 +- Parallel.Cli/Commands/VaultsCommand.cs | 31 +--- Parallel.Cli/Program.cs | 4 +- Parallel.Cli/Utils/CommandLine.cs | 5 +- Parallel.Cli/Utils/ProgressReport.cs | 15 +- .../Database/Contexts/SqliteContext.cs | 12 +- Parallel.Core/Database/DatabaseConnection.cs | 33 ---- .../Events/MessageRecievedEventArgs.cs | 1 + .../IO/FileSystem/DotNetFileSystem.cs | 111 ++++-------- .../IO/FileSystem/FileSystemManager.cs | 10 +- Parallel.Core/IO/FileSystem/IFileSystem.cs | 31 +--- Parallel.Core/IO/FileSystem/SftpFileSystem.cs | 170 ++++++------------ Parallel.Core/IO/PathBuilder.cs | 107 ++++++++--- Parallel.Core/IO/Recovery/RecoveryManager.cs | 86 --------- Parallel.Core/IO/Recovery/RecoveryPoint.cs | 55 ------ Parallel.Core/IO/Scanning/FileScanner.cs | 5 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 61 +++++-- Parallel.Core/IO/Syncing/DeltaSyncManager.cs | 4 +- Parallel.Core/IO/Syncing/FileSyncManager.cs | 23 ++- Parallel.Core/IO/Syncing/ISyncManager.cs | 16 +- Parallel.Core/IO/Syncing/SyncManager.cs | 6 +- Parallel.Core/Models/SystemFile.cs | 5 + Parallel.Core/Security/Encryption.cs | 5 +- Parallel.Core/Settings/DatabaseCredentials.cs | 45 ----- Parallel.Core/Settings/LocalVaultConfig.cs | 67 +++++++ Parallel.Core/Settings/ParallelConfig.cs | 101 +++++++++++ Parallel.Core/Settings/ParallelSettings.cs | 112 ------------ .../{VaultConfig.cs => RemoteVaultConfig.cs} | 103 +---------- 31 files changed, 476 insertions(+), 981 deletions(-) delete mode 100644 Parallel.Cli/Commands/DecryptCommand.cs delete mode 100644 Parallel.Cli/Commands/EncryptCommand.cs delete mode 100644 Parallel.Core/Database/DatabaseConnection.cs delete mode 100644 Parallel.Core/IO/Recovery/RecoveryManager.cs delete mode 100644 Parallel.Core/IO/Recovery/RecoveryPoint.cs delete mode 100644 Parallel.Core/Settings/DatabaseCredentials.cs create mode 100644 Parallel.Core/Settings/LocalVaultConfig.cs create mode 100644 Parallel.Core/Settings/ParallelConfig.cs delete mode 100644 Parallel.Core/Settings/ParallelSettings.cs rename Parallel.Core/Settings/{VaultConfig.cs => RemoteVaultConfig.cs} (52%) diff --git a/Parallel.Cli/Commands/DecryptCommand.cs b/Parallel.Cli/Commands/DecryptCommand.cs deleted file mode 100644 index b451fba..0000000 --- a/Parallel.Cli/Commands/DecryptCommand.cs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using System.CommandLine; -using System.Diagnostics; -using System.Text; -using Parallel.Cli.Utils; -using Parallel.Core.Database; -using Parallel.Core.IO; -using Parallel.Core.Models; -using Parallel.Core.Settings; -using Parallel.Core.Utils; - -namespace Parallel.Cli.Commands -{ - public class DecryptCommand : Command - { - private readonly Argument _sourceArg = new("path", "The source path of files to zip."); - private readonly Option _configOpt = new(["--config", "-c"], "The vault configuration to use."); - - private IDatabase? _database; - private Stopwatch _sw = new Stopwatch(); - private List _tasks = new List(); - private int _totalTasks = 0; - - public DecryptCommand() : base("decrypt", "Decrypts a file or directory.") - { - this.AddArgument(_sourceArg); - this.SetHandler(async (path, config) => - { - VaultConfig? vault = VaultConfig.Load(Program.Settings, config); - if (vault == null) - { - CommandLine.WriteLine("No active vault was found!", ConsoleColor.Yellow); - return; - } - - _database = DatabaseConnection.CreateNew(vault); - string masterKey = vault.FileSystem.EncryptionKey ?? throw new ArgumentException("No encryption key provided!"); - if (PathBuilder.IsDirectory(path)) - { - await DecryptDirectoryAsync(path, masterKey); - } - else if (PathBuilder.IsFile(path)) - { - CommandLine.WriteLine($"Decrypting {path}...", ConsoleColor.DarkGray); - await DecryptFileAsync(path, masterKey); - - CommandLine.WriteLine($"Successfully decrypted file: {path}", ConsoleColor.Green); - } - else - { - CommandLine.WriteLine("The specified path is invalid.", ConsoleColor.Red); - } - }, _sourceArg, _configOpt); - } - - private async Task DecryptDirectoryAsync(string path, string masterKey) - { - CommandLine.WriteLine($"Scanning for files in {path}...", ConsoleColor.DarkGray); - string[] files = Directory.EnumerateFiles(path, $"*", SearchOption.AllDirectories).ToArray(); - if (files.Length == 0) - { - CommandLine.WriteLine("No files found to decrypt!", ConsoleColor.Yellow); - return; - } - - CommandLine.WriteLine($"Decrypting {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); - _totalTasks = files.Length; - _tasks = files.Select(file => Task.Run(async () => - { - await DecryptFileAsync(file, masterKey); - CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw.Elapsed, ConsoleColor.DarkGray); - })).ToList(); - - await Task.WhenAll(_tasks); - CommandLine.WriteLine($"Successfully decrypted {files.Length.ToString("N0")} files in {_sw.Elapsed}.", ConsoleColor.Green); - } - - private async Task DecryptFileAsync(string path, string masterKey) - { - SystemFile systemFile = await _database?.GetFileAsync(path)! ?? new SystemFile(path); - if (File.Exists(systemFile.LocalPath) && systemFile.Encrypted) - { - string tempFile = Path.Combine(PathBuilder.TempDirectory, Path.GetFileName(systemFile.LocalPath)) + ".tmp"; - await using (FileStream openFile = new FileStream(systemFile.LocalPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - await using (FileStream createFile = new FileStream(tempFile, FileMode.OpenOrCreate)) - { - systemFile.Encrypted = false; - - Encryption.DecryptStream(openFile, createFile, masterKey, systemFile.LastWrite, systemFile.Salt, systemFile.IV); - if (!await _database?.AddFileAsync(systemFile)!) - { - CommandLine.WriteLine($"Failed to decrypt file: {systemFile.LocalPath}", ConsoleColor.Red); - if(File.Exists(tempFile)) File.Delete(tempFile); - return; - } - } - - File.Copy(tempFile, systemFile.LocalPath, true); - if(File.Exists(tempFile)) File.Delete(tempFile); - } - } - } -} \ No newline at end of file diff --git a/Parallel.Cli/Commands/EncryptCommand.cs b/Parallel.Cli/Commands/EncryptCommand.cs deleted file mode 100644 index f76caf0..0000000 --- a/Parallel.Cli/Commands/EncryptCommand.cs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using System.CommandLine; -using System.Diagnostics; -using System.IO.Compression; -using Parallel.Cli.Utils; -using Parallel.Core.Database; -using Parallel.Core.IO; -using Parallel.Core.Models; -using Parallel.Core.Security; -using Parallel.Core.Settings; -using Parallel.Core.Utils; - -namespace Parallel.Cli.Commands -{ - public class EncryptCommand : Command - { - private readonly Argument _sourceArg = new("path", "The source path to encrypt."); - private readonly Option _configOpt = new(["--config", "-c"], "The vault configuration to use."); - - private IDatabase? _database; - private Stopwatch _sw = new Stopwatch(); - private List _tasks = new List(); - private int _totalTasks = 0; - - public EncryptCommand() : base("encrypt", "Encrypts a file or directory.") - { - this.AddArgument(_sourceArg); - this.SetHandler(async (path, config) => - { - _sw = Stopwatch.StartNew(); - VaultConfig? vault = VaultConfig.Load(Program.Settings, config); - if (vault == null) - { - CommandLine.WriteLine("No active vault was found!", ConsoleColor.Yellow); - return; - } - - _database = DatabaseConnection.CreateNew(vault); - string masterKey = vault.FileSystem.EncryptionKey ?? throw new ArgumentException("No encryption key provided!"); - if (PathBuilder.IsDirectory(path)) - { - await EncryptDirectoryAsync(path, masterKey); - } - else if (PathBuilder.IsFile(path)) - { - CommandLine.WriteLine($"Encrypting {path}...", ConsoleColor.DarkGray); - await EncryptFileAsync(path, masterKey); - - CommandLine.WriteLine($"Successfully encrypted file: {path}", ConsoleColor.Green); - } - else - { - CommandLine.WriteLine("The specified path is invalid.", ConsoleColor.Red); - } - - }, _sourceArg, _configOpt); - } - - private async Task EncryptDirectoryAsync(string path, string masterKey) - { - CommandLine.WriteLine($"Scanning for files in {path}...", ConsoleColor.DarkGray); - string[] files = Directory.EnumerateFiles(path, $"*", SearchOption.AllDirectories).ToArray(); - if (files.Length == 0) - { - CommandLine.WriteLine("No files found to encrypt!", ConsoleColor.Yellow); - return; - } - - CommandLine.WriteLine($"Encrypting {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); - _totalTasks = files.Length; - _tasks = files.Select(file => Task.Run(async () => - { - await EncryptFileAsync(file, masterKey); - CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw.Elapsed, ConsoleColor.DarkGray); - })).ToList(); - - await Task.WhenAll(_tasks); - CommandLine.WriteLine($"Successfully encrypted {files.Length.ToString("N0")} files in {_sw.Elapsed}.", ConsoleColor.Green); - } - - private async Task EncryptFileAsync(string path, string masterKey) - { - SystemFile systemFile = await _database?.GetFileAsync(path)! ?? new SystemFile(path); - if (File.Exists(systemFile.LocalPath) && !systemFile.Encrypted) - { - string tempFile = Path.Combine(PathBuilder.TempDirectory, Path.GetFileName(systemFile.LocalPath)) + ".tmp"; - await using (FileStream openFile = new FileStream(systemFile.LocalPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - await using (FileStream createFile = new FileStream(tempFile, FileMode.OpenOrCreate)) - { - systemFile.Salt = HashGenerator.GenerateHash(16); - systemFile.IV = HashGenerator.GenerateHash(16); - systemFile.Encrypted = true; - - Encryption.EncryptStream(openFile, createFile, masterKey, systemFile.LastWrite, systemFile.Salt, systemFile.IV); - if (!await _database?.AddFileAsync(systemFile)!) - { - CommandLine.WriteLine($"Failed to encrypt file: {systemFile.LocalPath}", ConsoleColor.Red); - if(File.Exists(tempFile)) File.Delete(tempFile); - return; - } - } - - File.Copy(tempFile, systemFile.LocalPath, true); - if(File.Exists(tempFile)) File.Delete(tempFile); - } - } - } -} \ No newline at end of file diff --git a/Parallel.Cli/Commands/HistoryCommand.cs b/Parallel.Cli/Commands/HistoryCommand.cs index eabd7cb..e18e8bf 100644 --- a/Parallel.Cli/Commands/HistoryCommand.cs +++ b/Parallel.Cli/Commands/HistoryCommand.cs @@ -42,7 +42,7 @@ public HistoryCommand() : base("history", "Shows the history of files related to this.SetHandler((path, config, limit) => { CommandLine.WriteLine($"Retrieving backup information...", ConsoleColor.DarkGray); - IDatabase? db = DatabaseConnection.CreateNew(VaultConfig.Load(Program.Settings, config)); + IDatabase? db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); if (limit == 0) limit = Limit; DisplayHistories(db?.GetHistory(path, limit).ToArray()); @@ -54,7 +54,7 @@ public HistoryCommand() : base("history", "Shows the history of files related to _pushCmd.SetHandler((path, config, limit) => { CommandLine.WriteLine($"Retrieving backup information...", ConsoleColor.DarkGray); - IDatabase? db = DatabaseConnection.CreateNew(VaultConfig.Load(Program.Settings, config)); + IDatabase? db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); if (limit == 0) limit = Limit; DisplayHistories(db?.GetHistory(path, HistoryType.Pushed, limit).ToArray()); @@ -66,7 +66,7 @@ public HistoryCommand() : base("history", "Shows the history of files related to _deleteCmd.SetHandler((path, config, limit) => { CommandLine.WriteLine($"Retrieving archive information...", ConsoleColor.DarkGray); - IDatabase? db = DatabaseConnection.CreateNew(VaultConfig.Load(Program.Settings, config)); + IDatabase? db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); if (limit == 0) limit = Limit; DisplayHistories(db?.GetHistory(path, HistoryType.Archived, limit).ToArray()); @@ -78,7 +78,7 @@ public HistoryCommand() : base("history", "Shows the history of files related to _cleanCmd.SetHandler((path, config, limit) => { CommandLine.WriteLine($"Retrieving clean information...", ConsoleColor.DarkGray); - IDatabase? db = DatabaseConnection.CreateNew(VaultConfig.Load(Program.Settings, config)); + IDatabase? db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); if (limit == 0) limit = Limit; DisplayHistories(db?.GetHistory(path, HistoryType.Cleaned, limit).ToArray()); @@ -90,7 +90,7 @@ public HistoryCommand() : base("history", "Shows the history of files related to _cloneCmd.SetHandler((path, config, limit) => { CommandLine.WriteLine($"Retrieving clone information...", ConsoleColor.DarkGray); - IDatabase db = DatabaseConnection.CreateNew(VaultConfig.Load(Program.Settings, config)); + IDatabase db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); if (limit == 0) limit = Limit; DisplayHistories(db?.GetHistory(path, HistoryType.Cloned, limit).ToArray()); @@ -102,7 +102,7 @@ public HistoryCommand() : base("history", "Shows the history of files related to _pruneCmd.SetHandler((path, config, limit) => { CommandLine.WriteLine($"Retrieving prune information...", ConsoleColor.DarkGray); - IDatabase db = DatabaseConnection.CreateNew(VaultConfig.Load(Program.Settings, config)); + IDatabase db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); if (limit == 0) limit = Limit; DisplayHistories(db?.GetHistory(path, HistoryType.Pruned, limit).ToArray()); @@ -114,7 +114,7 @@ public HistoryCommand() : base("history", "Shows the history of files related to _pullCmd.SetHandler((path, config, limit) => { CommandLine.WriteLine($"Retrieving restore information...", ConsoleColor.DarkGray); - IDatabase db = DatabaseConnection.CreateNew(VaultConfig.Load(Program.Settings, config)); + IDatabase db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); if (limit == 0) limit = Limit; DisplayHistories(db?.GetHistory(path, HistoryType.Pulled, limit).ToArray()); diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 7d7961c..49e3bfc 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -47,10 +47,10 @@ private async Task SyncSystemAsync() private async Task SyncPathAsync(string path) { - await ParallelSettings.ForEachVaultAsync(async vault => + await ParallelConfig.ForEachVaultAsync(async vault => { ISyncManager sync = SyncManager.CreateNew(vault); - if (!sync.Initialize()) + if (!await sync.InitializeAsync()) { CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); return; @@ -87,7 +87,9 @@ await ParallelSettings.ForEachVaultAsync(async vault => CommandLine.WriteLine(vault, $"Backing up {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); await sync.PushFilesAsync(files, new ProgressReport(vault)); + CommandLine.WriteLine(vault, $"Successfully pushed {successFiles.ToString("N0")} files to '{vault.FileSystem.Address}'.", ConsoleColor.Green); + await sync.DisconnectAsync(); }); } } diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index 6413342..b1c8fa0 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -25,31 +25,17 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") this.SetHandler(() => { CommandLine.WriteLine("Active vaults:"); - Program.Settings.ForEachVault(vault => + for (int i = 0; i < Program.Settings.Vaults.Count; i++) { - CommandLine.WriteLine(vault.Name); - }); + LocalVaultConfig vault = Program.Settings.Vaults.ElementAt(i); + CommandLine.WriteLine($"{i + 1}: {vault.Name} ({vault.Id})"); + } }); this.AddCommand(addCmd); addCmd.SetHandler(() => { - CommandLine.WriteLine("Creating new database credentials...", ConsoleColor.DarkGray); - DatabaseCredentials dbc = new DatabaseCredentials(); - dbc.Provider = Enum.Parse(CommandLine.ReadString($"Provider ({string.Join(", ", Enum.GetNames(typeof(DatabaseProvider)))})"), true); - if (dbc.Provider == DatabaseProvider.Local) - { - dbc = DatabaseCredentials.Local; - } - else - { - dbc.Address = CommandLine.ReadString("Address"); - dbc.Username = CommandLine.ReadString("Username"); - dbc.Password = CommandLine.ReadPassword("Password"); - dbc.Name = CommandLine.ReadString("Name"); - } - - CommandLine.WriteLine("Creating new file system credentials...", ConsoleColor.DarkGray); + CommandLine.WriteLine("Creating new storage vault...", ConsoleColor.DarkGray); FileSystemCredentials fsc = new FileSystemCredentials(); fsc.Service = Enum.Parse(CommandLine.ReadString($"Service ({string.Join(", ", Enum.GetNames(typeof(FileService)))})"), true); if (fsc.Service == FileService.Local) @@ -74,10 +60,11 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") fsc.EncryptionKey = HashGenerator.GenerateHash(32, true); string? profileName = CommandLine.ReadString("Profile Name"); - VaultConfig vault = new VaultConfig(profileName, dbc, fsc); - vault.SaveToFile(); + LocalVaultConfig localVault = new LocalVaultConfig(profileName, fsc); + Program.Settings.Vaults.Add(localVault); + Program.Settings.Save(); - CommandLine.WriteLine($"Saved new connection vault: '{vault.Name}'"); + CommandLine.WriteLine($"Saved new storage vault: '{localVault.Name}' ({localVault.Id})"); }); this.AddCommand(setCmd); diff --git a/Parallel.Cli/Program.cs b/Parallel.Cli/Program.cs index 3c54b91..7369bd9 100644 --- a/Parallel.Cli/Program.cs +++ b/Parallel.Cli/Program.cs @@ -9,11 +9,11 @@ namespace Parallel.Cli { internal class Program { - internal static ParallelSettings Settings = new ParallelSettings(); + internal static ParallelConfig Settings = new ParallelConfig(); public static async Task Main(string[] args) { - Settings = ParallelSettings.Load(); + Settings = ParallelConfig.Load(); string logFile = Path.Combine(PathBuilder.ProgramData, "Logs", $"{DateTime.Now:MM-dd-yyyy hh-mm-ss}.log"); Log.Logger = new LoggerConfiguration().MinimumLevel.Debug().WriteTo.File(logFile).CreateLogger(); diff --git a/Parallel.Cli/Utils/CommandLine.cs b/Parallel.Cli/Utils/CommandLine.cs index 049cc89..90e9dfc 100644 --- a/Parallel.Cli/Utils/CommandLine.cs +++ b/Parallel.Cli/Utils/CommandLine.cs @@ -1,6 +1,7 @@ // Copyright 2025 Kyle Ebbinga using System.Text; +using Parallel.Core.Security; using Parallel.Core.Settings; using Parallel.Core.Utils; @@ -74,9 +75,9 @@ public static void Write(object value, ConsoleColor color = ConsoleColor.Gray) Console.ResetColor(); } - public static void WriteLine(VaultConfig vault, object value, ConsoleColor color = ConsoleColor.Gray) + public static void WriteLine(LocalVaultConfig localVault, object value, ConsoleColor color = ConsoleColor.Gray) { - string baseLog = $"[{vault.Id}] {value}"; + string baseLog = $"[{localVault.Id}] {value}"; switch(color) { default: diff --git a/Parallel.Cli/Utils/ProgressReport.cs b/Parallel.Cli/Utils/ProgressReport.cs index 9710575..f2c2f9b 100644 --- a/Parallel.Cli/Utils/ProgressReport.cs +++ b/Parallel.Cli/Utils/ProgressReport.cs @@ -6,17 +6,26 @@ namespace Parallel.Cli.Utils { - public class ProgressReport(VaultConfig vault) : IProgressReporter + public class ProgressReport : IProgressReporter { + private readonly LocalVaultConfig _localVault; + private readonly int _totalFiles; + + public ProgressReport(LocalVaultConfig localVault, int totalFiles) + { + _localVault = localVault; + _totalFiles = totalFiles; + } + public void Report(ProgressOperation operation, SystemFile file, int current, int total) { int percent = current * 100 / total; - CommandLine.WriteLine($"[{percent}%] <{vault.Id}> {operation}: {file.LocalPath}"); + CommandLine.WriteLine($"[{percent}%] <{_localVault.Id}> {operation}: {file.LocalPath}"); } public void Failed(Exception exception, SystemFile file) { - CommandLine.WriteLine(vault, $"Failed to upload file: '{file.LocalPath}'", ConsoleColor.Red); + CommandLine.WriteLine(_localVault, $"Failed to upload file: '{file.LocalPath}'", ConsoleColor.Red); } } } \ No newline at end of file diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index d0b4c2c..aaf6525 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -4,6 +4,7 @@ using System.Data; using System.Diagnostics; using Dapper; +using Parallel.Core.IO; using Parallel.Core.Models; using Parallel.Core.Settings; using Parallel.Core.Utils; @@ -21,10 +22,9 @@ public class SqliteContext : IDatabase /// /// /// - public SqliteContext(DatabaseCredentials credentials, string profileId) + public SqliteContext(LocalVaultConfig localVault) { - FilePath = credentials.Address; - ProfileId = profileId; + FilePath = PathBuilder.GetDatabaseFile(localVault); } #region Base @@ -38,13 +38,13 @@ public IDbConnection CreateConnection() /// public async Task InitializeAsync() { - Log.Information("Creating local database..."); + Log.Information("Creating index database..."); File.Create(FilePath).Close(); File.SetAttributes(FilePath, File.GetAttributes(FilePath) | FileAttributes.Hidden); using IDbConnection connection = CreateConnection(); - await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`vault` TEXT NOT NULL, `id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `encrypted` INTEGER NOT NULL DEFAULT 0, `salt` TEXT, `iv` TEXT, `checksum` TEXT, PRIMARY KEY(`vault`, `id`));"); - await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `history` (`vault` TEXT NOT NULL, `timestamp` LONG INTEGER NOT NULL, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`vault`, `timestamp`));"); + await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` TEXT, PRIMARY KEY(`id`));"); + await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `history` (`timestamp` LONG INTEGER NOT NULL, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`timestamp`));"); } #endregion diff --git a/Parallel.Core/Database/DatabaseConnection.cs b/Parallel.Core/Database/DatabaseConnection.cs deleted file mode 100644 index 3fd19f0..0000000 --- a/Parallel.Core/Database/DatabaseConnection.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using Parallel.Core.Settings; - -namespace Parallel.Core.Database -{ - /// - /// The supported file service types. - /// - public enum DatabaseProvider - { - Local - } - - /// - /// Represents a way to connect to different . - /// - public class DatabaseConnection - { - public static IDatabase? CreateNew(VaultConfig? vault) - { - switch(vault?.Database.Provider) - { - default: return null; - - case DatabaseProvider.Local: - IDatabase db = new SqliteContext(vault.Database, vault.Id); - if (!File.Exists(vault.Database.Address)) db.InitializeAsync(); - return db; - } - } - } -} \ No newline at end of file diff --git a/Parallel.Core/Events/MessageRecievedEventArgs.cs b/Parallel.Core/Events/MessageRecievedEventArgs.cs index a8ae445..aeab27d 100644 --- a/Parallel.Core/Events/MessageRecievedEventArgs.cs +++ b/Parallel.Core/Events/MessageRecievedEventArgs.cs @@ -2,6 +2,7 @@ using System.Net.Sockets; using System.Text; +using Parallel.Core.Security; using Parallel.Core.Utils; namespace Parallel.Core.Events diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index 941d721..d608956 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -17,17 +17,20 @@ namespace Parallel.Core.IO.FileSystem /// public class DotNetFileSystem : IFileSystem { - private readonly VaultConfig _vault; + private readonly LocalVaultConfig _vaultConfig; /// /// Represents an for interacting with physical machine hardware. /// - /// The vault to use. - public DotNetFileSystem(VaultConfig vault) + /// The vault to use. + public DotNetFileSystem(LocalVaultConfig vaultConfig) { - _vault = vault; + _vaultConfig = vaultConfig; } + /// + public void Dispose() { } + /// public Task CreateDirectoryAsync(string path) { @@ -54,66 +57,24 @@ public Task DeleteFileAsync(string path) return Task.CompletedTask; } - /// - public async Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress) + public Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress) { - if (!files.Any()) return; - for (int i = 0; i < files.Length; i++) - { - SystemFile file = files[i]; - Stopwatch sw = new Stopwatch(); - - progress.Report(ProgressOperation.Downloading, file, i, files.Length); - await using FileStream createStream = File.Create(file.LocalPath); - await using FileStream openStream = File.OpenRead(file.RemotePath); - await using GZipStream gzipStream = new GZipStream(openStream, CompressionMode.Decompress); - await gzipStream.CopyToAsync(createStream); - - Log.Debug($"Downloaded '{file.LocalPath}' in {sw.ElapsedMilliseconds}ms"); - } + throw new NotImplementedException(); } /// - public Task GetDirectoryNameAsync(string path) + public async Task DownloadFileAsync(string sourcePath, string destPath) { - return Task.FromResult(Path.GetDirectoryName(path)); + await using FileStream openStream = File.OpenRead(sourcePath); + await using FileStream createStream = File.Create(destPath); + await using GZipStream gzipStream = new GZipStream(openStream, CompressionMode.Decompress); + await gzipStream.CopyToAsync(createStream); } - /// - public Task> GetFilesAsync() - { - Dictionary files = new Dictionary(); - foreach (string file in Directory.GetFiles(PathBuilder.RootDirectory(_vault), "*.gz", SearchOption.AllDirectories)) - { - FileInfo fi = new(file); - files.Add(fi.FullName, new SystemFile(file) - { - Name = fi.Name, - RemotePath = fi.FullName, - LastWrite = new UnixTime(fi.LastWriteTime), - RemoteSize = fi.Length - }); - } - - return Task.FromResult(files); - } - - /// - public Task GetFilesAsync(string path) + /// + public Task ExistsAsync(string path) { - List list = new(); - foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories)) - { - FileInfo fi = new(file); - list.Add(new SystemFile(file) - { - Name = fi.Name, - RemotePath = fi.FullName, - RemoteSize = fi.Length - }); - } - - return Task.FromResult(list.ToArray()); + return Task.FromResult(Directory.Exists(path) || File.Exists(path)); } /// @@ -128,35 +89,29 @@ public Task GetFileAsync(string path) }); } - /// - public Task PingAsync() + public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress) { - Stopwatch sw = Stopwatch.StartNew(); - if (!Directory.Exists(PathBuilder.RootDirectory(_vault))) return Task.FromResult(-1); - return Task.FromResult(sw.ElapsedMilliseconds); + await System.Threading.Tasks.Parallel.ForEachAsync(files, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount / 2 }, async (file, ct) => + { + + }); } - /// - public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress) + /// + public async Task UploadFileAsync(string sourcePath, string destPath) { - if (!files.Any()) return; - await Task.WhenAll(files.Select(file => Task.Run(async () => - { - Stopwatch sw = new Stopwatch(); - file.RemotePath = PathBuilder.Remote(file.LocalPath, _vault); - if (File.Exists(file.RemotePath)) File.SetAttributes(file.RemotePath, ~FileAttributes.ReadOnly & File.GetAttributes(file.RemotePath)); - string parent = Path.GetDirectoryName(file.RemotePath); - if (!Directory.Exists(parent)) Directory.CreateDirectory(parent); - await using FileStream createStream = File.Create(file.RemotePath); - await using FileStream openStream = File.OpenRead(file.LocalPath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream); + if (await ExistsAsync(destPath)) File.SetAttributes(destPath, ~FileAttributes.ReadOnly & File.GetAttributes(destPath)); + string? parent = Path.GetDirectoryName(destPath); + if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); + + await using FileStream openStream = File.OpenRead(sourcePath); + await using FileStream createStream = File.Create(destPath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await openStream.CopyToAsync(gzipStream); - Log.Debug($"Uploaded '{file.RemotePath}' in {sw.ElapsedMilliseconds}ms"); - File.SetAttributes(file.RemotePath, File.GetAttributes(file.RemotePath) | FileAttributes.ReadOnly); - }))); + File.SetAttributes(destPath, File.GetAttributes(destPath) | FileAttributes.ReadOnly); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/FileSystemManager.cs b/Parallel.Core/IO/FileSystem/FileSystemManager.cs index 1e93b4d..adb410f 100644 --- a/Parallel.Core/IO/FileSystem/FileSystemManager.cs +++ b/Parallel.Core/IO/FileSystem/FileSystemManager.cs @@ -33,13 +33,13 @@ public static class FileSystemManager /// /// Creates a new file system association. /// - /// The vault needed for the associated file system. - public static IFileSystem CreateNew(VaultConfig vault) + /// The vault needed for the associated file system. + public static IFileSystem CreateNew(LocalVaultConfig vaultConfig) { - return vault.FileSystem.Service switch + return vaultConfig.FileSystem.Service switch { - FileService.Local => new DotNetFileSystem(vault), - FileService.Remote => new SftpFileSystem(vault), + FileService.Local => new DotNetFileSystem(vaultConfig), + FileService.Remote => new SftpFileSystem(vaultConfig), //FileService.Cloud => new AmazonS3FileSystem(credentials), _ => null }; diff --git a/Parallel.Core/IO/FileSystem/IFileSystem.cs b/Parallel.Core/IO/FileSystem/IFileSystem.cs index 7f043da..7346d81 100644 --- a/Parallel.Core/IO/FileSystem/IFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/IFileSystem.cs @@ -13,7 +13,7 @@ namespace Parallel.Core.IO.FileSystem /// /// Defines the way for communicating with a file system. /// - public interface IFileSystem + public interface IFileSystem : IDisposable { /// /// Creates all directories and subdirectories in the specified path unless they already exist. @@ -34,31 +34,18 @@ public interface IFileSystem Task DeleteFileAsync(string path); /// - /// Downloads a file from the associated file system. + /// Downloads an array of files from the associated file system. /// /// /// Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress); /// - /// Returns the parent directory name. + /// Checks if a path exists on the associated file system. /// /// - /// - Task GetDirectoryNameAsync(string path); - - /// - /// Gets all the files in the backup. - /// - /// A dictionary of s with the key being the backup path adn the value being the associated . - Task> GetFilesAsync(); - - /// - /// Gets all the files in the current directory. - /// - /// - /// A read-only collection of s. - Task GetFilesAsync(string path); + /// True if path exists, otherwise false. + Task ExistsAsync(string path); /// /// Gets a file on the associated file system. @@ -68,13 +55,7 @@ public interface IFileSystem Task GetFileAsync(string path); /// - /// Pings the remote file system. - /// - /// The time, in milliseconds, of the database latency. -1 if disconnected. - Task PingAsync(); - - /// - /// Uploads a file to the associated file system. + /// Uploads an array of files to the associated file system. /// /// /// diff --git a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs index 3a6a503..f00c498 100644 --- a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs @@ -8,6 +8,7 @@ using Newtonsoft.Json.Linq; using Parallel.Core.Diagnostics; using Parallel.Core.Models; +using Parallel.Core.Security; using Parallel.Core.Utils; namespace Parallel.Core.IO.FileSystem @@ -18,68 +19,59 @@ namespace Parallel.Core.IO.FileSystem public class SftpFileSystem : IFileSystem { private readonly ConnectionInfo _connectionInfo; - private readonly VaultConfig _vault; + private readonly SftpClient _client; /// /// Represents an for interacting with an SSH server. /// - /// The credentials to log in with. - public SftpFileSystem(VaultConfig vault) + /// The credentials to log in with. + public SftpFileSystem(LocalVaultConfig localVault) { - _connectionInfo = new ConnectionInfo(vault.FileSystem.Address, vault.FileSystem.Username, new PasswordAuthenticationMethod(vault.FileSystem.Username, Encryption.Decode(vault.FileSystem.Password))); - _vault = vault; + _connectionInfo = new ConnectionInfo(localVault.FileSystem.Address, localVault.FileSystem.Username, new PasswordAuthenticationMethod(localVault.FileSystem.Username, Encryption.Decode(localVault.FileSystem.Password))); + _client = new SftpClient(_connectionInfo); + _client.Connect(); + } + + + /// + public void Dispose() + { + if (_client.IsConnected) _client.Disconnect(); + _client.Dispose(); } /// public async Task CreateDirectoryAsync(string path) { - using (SftpClient sftp = new SftpClient(_connectionInfo)) + if (_client.IsConnected) { - sftp.Connect(); - if (sftp.IsConnected) + string parentDir = string.Empty; + foreach (string subPath in path.Split('/')) { - string parentDir = string.Empty; - foreach (string subPath in path.Split('/')) + parentDir += $"/{subPath}"; + if (!await _client.ExistsAsync(parentDir)) { - parentDir += $"/{subPath}"; - if (!await sftp.ExistsAsync(parentDir)) - { - await sftp.CreateDirectoryAsync(parentDir); - } + await _client.CreateDirectoryAsync(parentDir); } } - - sftp.Disconnect(); } } /// public async Task DeleteDirectoryAsync(string path) { - using (SftpClient sftp = new SftpClient(_connectionInfo)) + if (await ExistsAsync(path)) { - sftp.Connect(); - if (sftp.IsConnected && await sftp.ExistsAsync(path)) - { - await sftp.DeleteDirectoryAsync(path); - } - - sftp.Disconnect(); + await _client.DeleteDirectoryAsync(path); } } /// public async Task DeleteFileAsync(string path) { - using (SftpClient sftp = new SftpClient(_connectionInfo)) + if (await ExistsAsync(path)) { - sftp.Connect(); - if (sftp.IsConnected && await sftp.ExistsAsync(path)) - { - await sftp.DeleteAsync(path); - } - - sftp.Disconnect(); + await _client.DeleteAsync(path); } } @@ -88,116 +80,56 @@ public Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress) throw new NotImplementedException(); } - public Task GetDirectoryNameAsync(string path) - { - throw new NotImplementedException(); - } - - public Task> GetFilesAsync() + public async Task ExistsAsync(string path) { - throw new NotImplementedException(); - } - - /// - public async Task GetFilesAsync(string path) - { - List list = new(); - using (SftpClient sftp = new SftpClient(_connectionInfo)) - { - sftp.Connect(); - if (sftp.IsConnected && await sftp.ExistsAsync(path)) - { - foreach (ISftpFile file in sftp.ListDirectory(path)) - { - list.Add(new SystemFile(file.FullName) - { - Name = file.Name, - RemotePath = file.FullName, - RemoteSize = file.Length - }); - } - } - - sftp.Disconnect(); - } - - return list.ToArray(); + return _client.IsConnected && await _client.ExistsAsync(path); } /// public async Task GetFileAsync(string path) { SystemFile file = new SystemFile(path); - using (SftpClient sftp = new SftpClient(_connectionInfo)) + if (await ExistsAsync(path)) { - sftp.Connect(); - if (sftp.IsConnected && await sftp.ExistsAsync(path)) + ISftpFile sf = _client.Get(path); + file = new SystemFile(sf.FullName) { - ISftpFile sf = sftp.Get(path); - file = new SystemFile(sf.FullName) - { - Name = sf.Name, - RemotePath = sf.FullName, - RemoteSize = sf.Length, - }; - } - - sftp.Disconnect(); + Name = sf.Name, + RemoteSize = sf.Length, + }; } return file; } - /// - public async Task PingAsync() - { - CancellationTokenSource cts = new(); - Stopwatch sw = Stopwatch.StartNew(); - using (SftpClient sftp = new SftpClient(_connectionInfo)) - { - await sftp.ConnectAsync(cts.Token); - if (!sftp.IsConnected) return -1; - sftp.Disconnect(); - } - - return sw.ElapsedMilliseconds; - } - /// public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress) { - using SftpClient sftp = new SftpClient(_connectionInfo); - sftp.Connect(); - if (sftp.IsConnected) + for (int i = 0; i < files.Length; i++) { - for (int i = 0; i < files.Length; i++) - { - SystemFile file = files[i]; - Stopwatch sw = new Stopwatch(); - progress.Report(ProgressOperation.Uploading, file, i, files.Length); - if (await sftp.ExistsAsync(file.RemotePath)) sftp.ChangePermissions(file.RemotePath, 644); + SystemFile file = files[i]; + Stopwatch sw = new Stopwatch(); + progress.Report(ProgressOperation.Uploading, file, i, files.Length); + if (await _client.ExistsAsync(file.RemotePath)) _client.ChangePermissions(file.RemotePath, 644); - string parentDir = string.Empty; - foreach (string subPath in file.RemotePath.Split('/')) + string parentDir = string.Empty; + foreach (string subPath in file.RemotePath.Split('/')) + { + parentDir += $"/{subPath}"; + if (!await _client.ExistsAsync(parentDir)) { - parentDir += $"/{subPath}"; - if (!await sftp.ExistsAsync(parentDir)) - { - await sftp.CreateDirectoryAsync(parentDir); - } + await _client.CreateDirectoryAsync(parentDir); } + } - await using SftpFileStream createStream = sftp.Create(file.RemotePath); - await using FileStream openStream = File.OpenRead(file.LocalPath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream); - sftp.ChangePermissions(file.RemotePath, 444); + await using SftpFileStream createStream = _client.Create(file.RemotePath); + await using FileStream openStream = File.OpenRead(file.LocalPath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await openStream.CopyToAsync(gzipStream); + _client.ChangePermissions(file.RemotePath, 444); - Log.Debug($"Uploaded '{file.RemotePath}' in {sw.ElapsedMilliseconds}ms"); - } + Log.Debug($"Uploaded '{file.RemotePath}' in {sw.ElapsedMilliseconds}ms"); } - - sftp.Disconnect(); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index 99fe668..db16c8a 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -1,6 +1,8 @@ // Copyright 2025 Kyle Ebbinga using System.Runtime.InteropServices; +using System.Text; +using System.Text.RegularExpressions; using Parallel.Core.IO.FileSystem; using Parallel.Core.Models; using Parallel.Core.Settings; @@ -9,15 +11,17 @@ namespace Parallel.Core.IO { /// - /// Represents the way to build paths on different operating systems. + /// Represents the way to build paths on different operating systems. This class cannot be inherited. /// public class PathBuilder { + private static readonly Regex DriveLetterRegex = new(@"^[a-zA-Z]:", RegexOptions.Compiled); + public static string TempDirectory { get { - string tempFolder = Path.Combine(Path.GetTempPath(), $"parallel_{UnixTime.Now.TotalSeconds}"); + string tempFolder = Path.Combine(Path.GetTempPath(), "Parallel"); if (!Directory.Exists(tempFolder)) Directory.CreateDirectory(tempFolder); return tempFolder; } @@ -50,37 +54,90 @@ public static string ProgramData } /// - /// Builds the path for the local file system. + /// Combines an array of strings into a path. This differs from by using the string context for combining paths instead of using the path operator environment variable. /// - /// - /// + /// /// - public static string Local(string path, FileSystemCredentials credentials) + public static string Combine(params string[] paths) { - string root = Path.Combine(credentials.RootDirectory, "Parallel", Environment.MachineName); - string main = path.Replace("/", "\\").Replace(root, string.Empty).Replace(".gz", string.Empty); + ArgumentNullException.ThrowIfNull(paths); + if (paths.Length == 0) return string.Empty; - Console.WriteLine(root); - Console.WriteLine(main); + // Detect context from the first path + bool isWindowsStyle = DriveLetterRegex.IsMatch(paths[0]); - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + char separator = isWindowsStyle ? '\\' : '/'; + char altSeparator = isWindowsStyle ? '/' : '\\'; + + StringBuilder sb = new StringBuilder(); + foreach (string p in paths) { - return main.Substring(1, main.Length - 1).Insert(1, ":"); + if (string.IsNullOrWhiteSpace(p)) continue; + + string part = p.Replace(altSeparator, separator); + + if (sb.Length == 0) + { + sb.Append(part.TrimEnd(separator)); + } + else + { + sb.Append(separator); + sb.Append(part.Trim(separator)); + } } - return main.Replace(@"\", "/"); + return sb.ToString(); } - public static string RootDirectory(VaultConfig vault) + /// + /// Gets the root directory of the vault. + /// + /// + /// + public static string GetRootDirectory(LocalVaultConfig localVault) { - string root = Path.Combine(vault.FileSystem.RootDirectory, "Parallel", vault.Id); - Log.Debug($"Root directory: {root}"); - return vault.FileSystem.Service switch - { - FileService.Local => root, - FileService.Remote => root.Replace('\\', '/'), - _ => string.Empty - }; + return Combine(localVault.FileSystem.RootDirectory, "Parallel", localVault.Id); + } + + /// + /// Gets the primary location where files are stored in the vault. + /// + /// + /// + public static string GetFilesDirectory(LocalVaultConfig localVault) + { + return Combine(GetRootDirectory(localVault), "Files"); + } + + /// + /// Gets the location where snapshots are stored in the vault. + /// + /// + /// + public static string GetSnapshotsDirectory(LocalVaultConfig localVault) + { + return Combine(GetRootDirectory(localVault), "Snapshots"); + } + + /// + /// Gets the path to the vault's configuration file. + /// + /// + /// + public static string GetConfigurationFile(LocalVaultConfig localVault) + { + return Combine(GetRootDirectory(localVault), "config.json"); + } + + /// + /// Gets the path to the vault's database file. + /// + /// + /// + public static string GetDatabaseFile(LocalVaultConfig localVault) + { + return Combine(GetRootDirectory(localVault), "index.db"); } /// @@ -89,10 +146,10 @@ public static string RootDirectory(VaultConfig vault) /// /// /// - public static string Remote(string path, VaultConfig vault) + public static string Remote(string path, RemoteVaultConfig remoteVaultConfig) { - string root = Path.Combine(vault.FileSystem.RootDirectory, "Parallel", vault.Id, "Files", path.Replace(":", string.Empty)) + ".gz"; - return vault.FileSystem.Service switch + string root = Path.Combine(remoteVaultConfig.FileSystem.RootDirectory, "Parallel", remoteVaultConfig.Id, "Files", path.Replace(":", string.Empty)) + ".gz"; + return remoteVaultConfig.FileSystem.Service switch { FileService.Local => root, FileService.Remote => root.Replace('\\', '/'), diff --git a/Parallel.Core/IO/Recovery/RecoveryManager.cs b/Parallel.Core/IO/Recovery/RecoveryManager.cs deleted file mode 100644 index 43f1ad7..0000000 --- a/Parallel.Core/IO/Recovery/RecoveryManager.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using System.Data; -using Parallel.Core.Database; -using Parallel.Core.IO.FileSystem; -using Parallel.Core.Models; -using Parallel.Core.Settings; - -namespace Parallel.Core.IO.Recovery -{ - /// - /// Represents the way to manage recovery points on the system. - /// - public class RecoveryManager - { - private readonly string _dbPath = Path.Combine(PathBuilder.ProgramData, Environment.MachineName + ".db"); - - public IDatabase Database { get; set; } - public IFileSystem FileSystem { get; set; } - public VaultConfig Vault { get; set; } - public string MachineName { get; } = Environment.MachineName; - public string RootFolder { get; set; } - - /// - /// Initializes a new instance of the class. - /// - /// - public RecoveryManager(VaultConfig vault) - { - Vault = vault; - Database = DatabaseConnection.CreateNew(vault); - FileSystem = FileSystemManager.CreateNew(vault); - } - - public bool Initialize() - { - try - { - Database = DatabaseConnection.CreateNew(Vault); - bool fsInit = (FileSystem != null) && FileSystem.PingAsync().Result >= 0; - Vault.IgnoreDirectories.Add(Vault.FileSystem.RootDirectory); - if (Vault != null) Vault.SaveToFile(); - return fsInit; - } - catch (Exception ex) - { - Log.Error(ex.GetBaseException().ToString()); - return false; - } - } - - /// - /// Loads a to restore the - /// - /// - public void Load(RecoveryPoint recoveryPoint) - { - - } - - /// - /// Saves the current file system state as a . - /// - /// - public RecoveryPoint Save() - { - /*RecoveryPoint rp = new(Profile.BackupDirectories, Profile.IgnoreDirectories); - DataTable dt = Database.GetFiles(); - foreach (DataRow row in dt.Rows) - { - SystemFile lf = new SystemFile(row); - if (lf.Deleted) - { - rp.DeletedFiles.Add(lf); - } - else - { - rp.LocalFiles.Add(lf); - } - } - - return rp;*/ - return new RecoveryPoint(ArraySegment.Empty, ArraySegment.Empty); - } - } -} \ No newline at end of file diff --git a/Parallel.Core/IO/Recovery/RecoveryPoint.cs b/Parallel.Core/IO/Recovery/RecoveryPoint.cs deleted file mode 100644 index ea57c4d..0000000 --- a/Parallel.Core/IO/Recovery/RecoveryPoint.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using Parallel.Core.Models; - -namespace Parallel.Core.IO.Recovery -{ - /// - /// Represents a collection of files at an instance of time on the local machine. - /// - public class RecoveryPoint - { - /// - /// The unique identifier. - /// - public string Id { get; } - - /// - /// The time of creation. - /// - public DateTime CreatedAt { get; } - - /// - /// An array of folders to back up. - /// - public string[] BackupFolders { get; } - - /// - /// An array of folders to ignore. - /// - public string[] IgnoreFolders { get; } - - /// - /// A collection of files that exist in the local machine. - /// - public List LocalFiles { get; } - - /// - /// A collection of deleted files that don't exist on the local machine. - /// - public List DeletedFiles { get; } - - /// - /// Initializes a new instance of the class. - /// - public RecoveryPoint(IEnumerable backupFolders, IEnumerable ignoreFolders) - { - Id = Guid.NewGuid().ToString(); - CreatedAt = DateTime.Now; - BackupFolders = backupFolders.ToArray(); - IgnoreFolders = ignoreFolders.ToArray(); - LocalFiles = new List(); - DeletedFiles = new List(); - } - } -} \ No newline at end of file diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index 699af81..aea7a6b 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -19,18 +19,15 @@ namespace Parallel.Core.IO.Scanning /// public class FileScanner { - private readonly VaultConfig _vault; private readonly IDatabase _db; - public FileScanner(VaultConfig vault, IDatabase database) + public FileScanner(IDatabase database) { - _vault = vault; _db = database; } public FileScanner(ISyncManager sync) { - _vault = sync.Vault; _db = sync.Database; } diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index c2f3362..d39ea53 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -2,7 +2,6 @@ using Parallel.Core.Database; using Parallel.Core.Diagnostics; -using Parallel.Core.IO.Backup; using Parallel.Core.IO.FileSystem; using Parallel.Core.Models; using Parallel.Core.Settings; @@ -14,8 +13,15 @@ namespace Parallel.Core.IO.Syncing /// public abstract class BaseSyncManager : ISyncManager { + protected string TempDirectory = PathBuilder.TempDirectory; + protected string TempConfigFile => Path.Combine(TempDirectory, $"{LocalVault.Id}.json"); + protected string TempDbFile => Path.Combine(TempDirectory, $"{LocalVault.Id}.db"); + + /// + public LocalVaultConfig LocalVault { get; private set; } + /// - public VaultConfig Vault { get; } + public RemoteVaultConfig RemoteVault { get; private set; } /// public IDatabase Database { get; set; } @@ -26,23 +32,45 @@ public abstract class BaseSyncManager : ISyncManager /// /// /// - /// - public BaseSyncManager(VaultConfig vault) + /// + public BaseSyncManager(LocalVaultConfig localVault) + { + FileSystem = FileSystemManager.CreateNew(localVault); + LocalVault = localVault; + } + + /// + public void Dispose() { - FileSystem = FileSystemManager.CreateNew(vault); - Vault = vault; + FileSystem.Dispose(); } /// - public virtual bool Initialize() + public async Task InitializeAsync() { try { - Database = DatabaseConnection.CreateNew(Vault); - FileSystem.CreateDirectoryAsync(PathBuilder.RootDirectory(Vault)); - Vault.IgnoreDirectories.Add(Vault.FileSystem.RootDirectory); - Vault.SaveToFile(); - return FileSystem.PingAsync().Result >= 0; + string root = PathBuilder.GetRootDirectory(LocalVault); + if (!await FileSystem.ExistsAsync(root)) + { + // Creates the root directory and default configuration. + await FileSystem.CreateDirectoryAsync(root); + RemoteVault = new RemoteVaultConfig(LocalVault); + } + else + { + SystemFile[] files = + [ + new SystemFile(TempConfigFile) { RemotePath = PathBuilder.GetConfigurationFile(LocalVault) }, + new SystemFile(TempDbFile) { RemotePath = PathBuilder.GetDatabaseFile(LocalVault) }, + ]; + + await FileSystem.DownloadFilesAsync(files, new ProgressLogger()); + } + + Database = new SqliteContext(LocalVault); + RemoteVault.IgnoreDirectories.Add(PathBuilder.GetRootDirectory(LocalVault)); + return true; } catch (Exception ex) { @@ -51,6 +79,15 @@ public virtual bool Initialize() } } + /// + public Task DisconnectAsync() + { + string configFile = PathBuilder.GetConfigurationFile(LocalVault); + + FileSystem.Dispose(); + return Task.CompletedTask; + } + /// public abstract Task PushFilesAsync(SystemFile[] files, IProgressReporter progress); diff --git a/Parallel.Core/IO/Syncing/DeltaSyncManager.cs b/Parallel.Core/IO/Syncing/DeltaSyncManager.cs index 10663d6..a9155a8 100644 --- a/Parallel.Core/IO/Syncing/DeltaSyncManager.cs +++ b/Parallel.Core/IO/Syncing/DeltaSyncManager.cs @@ -14,8 +14,8 @@ public class DeltaSyncManager : BaseSyncManager /// /// Initializes a new instance of the class. /// - /// - public DeltaSyncManager(VaultConfig vault) : base(vault) { } + /// + public DeltaSyncManager(RemoteVaultConfig remoteVaultConfig) : base(remoteVaultConfig) { } /// public override Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 57ecf45..66cfe71 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -16,14 +16,14 @@ namespace Parallel.Core.IO.Backup /// public class FileSyncManager : BaseSyncManager { - private List _tasks = new List(); - private int _totalFiles; - /// /// Initializes a new instance of the class. /// - /// - public FileSyncManager(VaultConfig vault) : base(vault) { } + /// + public FileSyncManager(LocalVaultConfig localVault) : base(localVault) + { + + } /// public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) @@ -32,8 +32,12 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter SystemFile[] backupFiles = files.Where(f => !f.Deleted).ToArray(); Log.Information($"Backing up {backupFiles.Length} files..."); await FileSystem.UploadFilesAsync(backupFiles, progress); + await System.Threading.Tasks.Parallel.ForEachAsync(files, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount / 2 }, async (file, ct) => + { + + }); + - Console.WriteLine($"Successfully pushed {backupFiles.Length} files.", ConsoleColor.Green); for (int i = 0; i < files.Length; i++) { SystemFile file = files.ElementAt(i); @@ -64,13 +68,6 @@ public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter if (!restoreFiles.Any()) return; await FileSystem.DownloadFilesAsync(restoreFiles, progress); - - for (int i = 0; i < files.Length; i++) - { - SystemFile file = files[i]; - Log.Information($"Restoring file: {file.LocalPath}..."); - file.RemotePath = PathBuilder.Remote(file.LocalPath, Vault); - } } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index 21a5641..9092bce 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -11,12 +11,17 @@ namespace Parallel.Core.IO.Syncing /// /// Defines the methods needed for backing up a file system. /// - public interface ISyncManager + public interface ISyncManager : IDisposable { /// - /// The back-up connection vault. + /// Gets the local vault configuration. /// - public VaultConfig Vault { get; } + public LocalVaultConfig LocalVault { get; } + + /// + /// Gets the remote vault configuration. + /// + public RemoteVaultConfig RemoteVault { get; } /// /// The associated database connection. @@ -29,10 +34,9 @@ public interface ISyncManager IFileSystem FileSystem { get; set; } /// - /// Initializes the backup manager by logging into the and + /// Initializes the associated and downloads the needed files. /// - /// - bool Initialize(); + Task InitializeAsync(); /// /// Pushes an array of files to a vault. diff --git a/Parallel.Core/IO/Syncing/SyncManager.cs b/Parallel.Core/IO/Syncing/SyncManager.cs index 4b5124f..49f4b8c 100644 --- a/Parallel.Core/IO/Syncing/SyncManager.cs +++ b/Parallel.Core/IO/Syncing/SyncManager.cs @@ -19,11 +19,11 @@ public static class SyncManager /// /// Creates a new instance of an . /// - /// + /// /// - public static ISyncManager CreateNew(VaultConfig vault) + public static ISyncManager CreateNew(LocalVaultConfig localVault) { - return new FileSyncManager(vault); + return new FileSyncManager(localVault); } } } \ No newline at end of file diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index 514abbd..5e2a144 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -156,6 +156,11 @@ public SystemFile(string vault, string id, string name, string localpath, string CheckSum = checksum; } + /// + /// Determines if this instance and another have the same values. + /// + /// + /// True if equal, otherwise false. public bool Equals(SystemFile value) { bool?[] results = diff --git a/Parallel.Core/Security/Encryption.cs b/Parallel.Core/Security/Encryption.cs index 3a7fc45..d49edcf 100644 --- a/Parallel.Core/Security/Encryption.cs +++ b/Parallel.Core/Security/Encryption.cs @@ -2,10 +2,9 @@ using System.Security.Cryptography; using System.Text; -using Parallel.Core.Models; -using Parallel.Core.Security; +using Parallel.Core.Utils; -namespace Parallel.Core.Utils +namespace Parallel.Core.Security { /// /// Provides functionality for encryption. This class cannot be inherited. diff --git a/Parallel.Core/Settings/DatabaseCredentials.cs b/Parallel.Core/Settings/DatabaseCredentials.cs deleted file mode 100644 index 761427b..0000000 --- a/Parallel.Core/Settings/DatabaseCredentials.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using Parallel.Core.Database; -using Parallel.Core.IO; - -namespace Parallel.Core.Settings -{ - /// - /// Represents credentials used to gain access to various s. - /// - public class DatabaseCredentials - { - /// - /// The associated provider of this database. - /// - public DatabaseProvider Provider { get; set; } = DatabaseProvider.Local; - - /// - /// The hostname or address of the database. - /// If using a , this will be a file path. - /// - public string Address { get; set; } = string.Empty; - - /// - /// The username of the database. - /// - public string? Username { get; set; } - - /// - /// The password of the database. - /// - public string? Password { get; set; } - - /// - /// The database name. - /// - public string Name { get; set; } = string.Empty; - - public static DatabaseCredentials Local => new() - { - Provider = DatabaseProvider.Local, - Address = Path.Combine(PathBuilder.ProgramData, $"{Environment.MachineName}.db") - }; - } -} \ No newline at end of file diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs new file mode 100644 index 0000000..a5a405d --- /dev/null +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -0,0 +1,67 @@ +// Copyright 2025 Kyle Ebbinga + +using Parallel.Core.Security; + +namespace Parallel.Core.Settings +{ + /// + /// Represents a localized vault connection configuration. + /// + public class LocalVaultConfig + { + /// + /// A unique hash used to identify the vault. + /// + public string Id { get; } = HashGenerator.GenerateHash(12, true); + + /// + /// The name of the vault. + /// + public string Name { get; set; } = "Default"; + + /// + /// The credentials needed to log in to the associated . + /// + public FileSystemCredentials FileSystem { get; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// + /// + [JsonConstructor] + public LocalVaultConfig(string id, string name, FileSystemCredentials fileSystem) + { + Id = id; + Name = name; + FileSystem = fileSystem; + } + + public LocalVaultConfig(string name, FileSystemCredentials fileSystem) + { + Id = HashGenerator.GenerateHash(12, true); + Name = name; + FileSystem = fileSystem; + } + + /// + /// Loads settings from a file. + /// + public static LocalVaultConfig? Load(string path) + { + return !File.Exists(path) ? null : JsonConvert.DeserializeObject(File.ReadAllText(path)); + } + + /// + /// Saves credentials to a file. + /// + /// + /// + public static void Save(ParallelConfig config, LocalVaultConfig localVault) + { + config.Vaults.Add(localVault); + config.Save(); + } + } +} \ No newline at end of file diff --git a/Parallel.Core/Settings/ParallelConfig.cs b/Parallel.Core/Settings/ParallelConfig.cs new file mode 100644 index 0000000..2083801 --- /dev/null +++ b/Parallel.Core/Settings/ParallelConfig.cs @@ -0,0 +1,101 @@ +// Copyright 2025 Kyle Ebbinga + +using Newtonsoft.Json; +using Org.BouncyCastle.Math.EC; +using Parallel.Core.IO; + +namespace Parallel.Core.Settings +{ + /// + /// + /// + public class ParallelConfig + { + /// + /// The location to the application configuration file. + /// + private static string ConfigFile { get; } = Path.Combine(PathBuilder.ProgramData, "Configuration.json"); + + /// + /// The location of files for different file system credentials./>. + /// + public static string VaultsDir { get; } = Path.Combine(PathBuilder.ProgramData, "Vaults"); + + // /// + // /// The address that will accept incoming commands. + // /// Default: 127.0.0.1 + // /// + // public string Address { get; set; } = "127.0.0.1"; + // + // /// + // /// The port number to listen for commands on. + // /// Default: 8192 + // /// + // public int ListenerPort { get; set; } = 8192; + + /// + /// Gets or sets the maximum number of concurrent vaults that can run. + /// Default: 2 + /// + public int MaxConcurrentVaults { get; set; } = 2; + + /// + /// Gets or sets the maximum number of concurrent processes that can run. + /// Default: Half the processor count. + /// + public int MaxConcurrentProcesses { get; set; } = Environment.ProcessorCount / 2; + + /// + /// The profiles to use. + /// When pulling, the CLI defaults to the first in the list. + /// + public HashSet Vaults { get; } = []; + + + /// + /// Loads settings from a file. + /// + public static ParallelConfig Load() + { + Log.Debug($"Loading config file: {ConfigFile}"); + if (File.Exists(ConfigFile)) + { + string json = File.ReadAllText(ConfigFile); + return JsonConvert.DeserializeObject(json); + } + else + { + return new ParallelConfig(); + } + } + + /// + /// Saves settings to a file. + /// + public void Save() + { + Log.Debug($"Saving config file: {ConfigFile}"); + if (!Directory.Exists(PathBuilder.ProgramData)) Directory.CreateDirectory(PathBuilder.ProgramData); + File.WriteAllText(ConfigFile, JsonConvert.SerializeObject(this, Formatting.Indented)); + } + + /// + /// Asynchronously runs an for each using the limiter. + /// + /// + /// + public async Task ForEachVaultAsync(Func actionAsync, CancellationToken cancellationToken = default) + { + ParallelOptions options = new ParallelOptions + { + MaxDegreeOfParallelism = MaxConcurrentVaults, + CancellationToken = cancellationToken + }; + + await System.Threading.Tasks.Parallel.ForEachAsync(Vaults, options, async (vault, ct) => + { + await actionAsync(vault); + }); + } + } +} \ No newline at end of file diff --git a/Parallel.Core/Settings/ParallelSettings.cs b/Parallel.Core/Settings/ParallelSettings.cs deleted file mode 100644 index d323473..0000000 --- a/Parallel.Core/Settings/ParallelSettings.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using Newtonsoft.Json; -using Org.BouncyCastle.Math.EC; -using Parallel.Core.IO; - -namespace Parallel.Core.Settings -{ - /// - /// - /// - public class ParallelSettings - { - /// - /// The location to the application configuration file. - /// - private static string ConfigFile { get; } = Path.Combine(PathBuilder.ProgramData, "settings.json"); - - /// - /// The location of files for different file system credentials./>. - /// - public static string VaultsDir { get; } = Path.Combine(PathBuilder.ProgramData, "Vaults"); - - /// - /// The address that will accept incoming commands. - /// Default: 127.0.0.1 - /// - public string Address { get; set; } = "127.0.0.1"; - - /// - /// The port number to listen for commands on. - /// Default: 8192 - /// - public int ListenerPort { get; set; } = 8192; - - /// - /// The profiles to use. - /// The CLI defaults to the first in the list. - /// - public HashSet Vaults { get; } = new HashSet(); - - - /// - /// Loads settings from a file. - /// - public static ParallelSettings Load() - { - Log.Debug($"Loading config file: {ConfigFile}"); - if (File.Exists(ConfigFile)) - { - string json = File.ReadAllText(ConfigFile); - return JsonConvert.DeserializeObject(json); - } - else - { - return new ParallelSettings(); - } - } - - /// - /// Saves settings to a file. - /// - public void Save() - { - Log.Debug($"Saving config file: {ConfigFile}"); - if (!Directory.Exists(PathBuilder.ProgramData)) Directory.CreateDirectory(PathBuilder.ProgramData); - File.WriteAllText(ConfigFile, JsonConvert.SerializeObject(this, Formatting.Indented)); - } - - /// - /// - /// - /// - public void ForEachVault(Action action) - { - foreach (string path in Directory.GetFiles(VaultsDir, "*.json", SearchOption.TopDirectoryOnly)) - { - VaultConfig? vault = VaultConfig.Load(path); - if (vault != null) action(vault); - } - } - - /// - /// Asynchronously runs an for each with a default of 3 at a time. - /// - /// - /// - public static async Task ForEachVaultAsync(Func actionAsync, int maxDegreeOfParallelism = 3) - { - string[] vaultPaths = Directory.GetFiles(VaultsDir, "*.json", SearchOption.TopDirectoryOnly); - SemaphoreSlim semaphore = new SemaphoreSlim(maxDegreeOfParallelism); - IEnumerable tasks = vaultPaths.Select(path => Task.Run(async () => - { - await semaphore.WaitAsync(); - try - { - VaultConfig? vault = VaultConfig.Load(path); - if (vault != null) - { - await actionAsync(vault); - } - } - finally - { - semaphore.Release(); - } - })); - - await Task.WhenAll(tasks); - } - } -} \ No newline at end of file diff --git a/Parallel.Core/Settings/VaultConfig.cs b/Parallel.Core/Settings/RemoteVaultConfig.cs similarity index 52% rename from Parallel.Core/Settings/VaultConfig.cs rename to Parallel.Core/Settings/RemoteVaultConfig.cs index c1a480d..a9dab5a 100644 --- a/Parallel.Core/Settings/VaultConfig.cs +++ b/Parallel.Core/Settings/RemoteVaultConfig.cs @@ -12,30 +12,10 @@ namespace Parallel.Core.Settings { /// - /// Represents a back-up connection. + /// Represents the configuration for the vault. /// - public class VaultConfig + public class RemoteVaultConfig : LocalVaultConfig { - /// - /// A unique hash used to identify the vault. - /// - public string Id { get; } = HashGenerator.GenerateHash(12, true); - - /// - /// The name of the vault. - /// - public string Name { get; set; } = "Default"; - - /// - /// The credentials needed to log in to the associated . - /// - public DatabaseCredentials Database { get; } - - /// - /// The credentials needed to log in to the associated . - /// - public FileSystemCredentials FileSystem { get; } - /// /// The amount of time, in minutes, between backup cycles. /// Default: 60 minutes @@ -78,84 +58,11 @@ public class VaultConfig /// Recommended when using a cloud-based to save on storage costs. /// Default: Empty /// - public HashSet PruneDirectories { get; } = new HashSet(); - - - /// - /// Initializes a new instance of the class. - /// - /// - /// - /// - /// - [JsonConstructor] - public VaultConfig(string id, string name, DatabaseCredentials database, FileSystemCredentials fileSystem) - { - Id = id; - Name = name; - Database = database; - FileSystem = fileSystem; - } - - /// - /// Initializes a new instance of the class. - /// - /// - /// - /// - public VaultConfig(string name, DatabaseCredentials database, FileSystemCredentials fileSystem) - { - Id = HashGenerator.GenerateHash(12, true); - Name = name; - Database = database; - FileSystem = fileSystem; - } - - /// - /// Loads settings from a file. - /// - public static VaultConfig? Load(string path) - { - if (!File.Exists(path)) return null; - string json = File.ReadAllText(path); - return JsonConvert.DeserializeObject(json); - } - - /// - /// Loads credentials from the app configuration. - /// - /// A instance. - public static VaultConfig? Load(ParallelSettings settings, string name) - { - VaultConfig? vault = Load(settings.Vaults.First()); - return string.IsNullOrEmpty(name) ? vault : Load(Path.Combine(ParallelSettings.VaultsDir, name + ".json")); - } - - /// - /// Saves credentials to a file. - /// - /// The current vault to save. - public static void Save(VaultConfig vault) - { - if (!Directory.Exists(ParallelSettings.VaultsDir)) Directory.CreateDirectory(ParallelSettings.VaultsDir); - string path = Path.Combine(ParallelSettings.VaultsDir, vault.Name + ".json"); - Log.Debug($"Saving vault file: {path}"); - if (!File.Exists(path)) - { - Log.Debug("Creating file -> " + path); - File.Create(path).Close(); - } + public HashSet PruneDirectories { get; } = []; - File.WriteAllText(path, JsonConvert.SerializeObject(vault, Formatting.Indented)); - } + public RemoteVaultConfig(LocalVaultConfig localVault) : base(localVault.Id, localVault.Name, localVault.FileSystem) { } - /// - /// Saves the current instance to a file. - /// - public void SaveToFile() - { - Save(this); - } + public RemoteVaultConfig(string profileName, FileSystemCredentials fsc) : base(profileName, fsc) { } #region Privates From 637c1fba3caea17a6fbc9a8cc646bd2e3dea4dd7 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 2 Sep 2025 02:58:22 -0500 Subject: [PATCH 2/8] IT BUILDS --- Parallel.Cli/Commands/HistoryCommand.cs | 139 ------------------ Parallel.Cli/Commands/PushCommand.cs | 17 +-- Parallel.Cli/Program.cs | 5 +- .../Database/Contexts/SqliteContext.cs | 6 +- .../IO/FileSystem/DotNetFileSystem.cs | 47 +++--- Parallel.Core/IO/PathBuilder.cs | 5 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 67 +++++---- Parallel.Core/IO/Syncing/FileSyncManager.cs | 3 + Parallel.Core/IO/Syncing/ISyncManager.cs | 11 +- Parallel.Core/Models/SystemFile.cs | 2 + Parallel.Core/Settings/LocalVaultConfig.cs | 2 +- Parallel.Core/Settings/ParallelConfig.cs | 7 +- Parallel.Core/Settings/RemoteVaultConfig.cs | 26 ++++ 13 files changed, 122 insertions(+), 215 deletions(-) delete mode 100644 Parallel.Cli/Commands/HistoryCommand.cs diff --git a/Parallel.Cli/Commands/HistoryCommand.cs b/Parallel.Cli/Commands/HistoryCommand.cs deleted file mode 100644 index e18e8bf..0000000 --- a/Parallel.Cli/Commands/HistoryCommand.cs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2025 Entex Interactive, LLC - -using System.CommandLine; -using System.Data; -using Parallel.Cli.Utils; -using Parallel.Core.Database; -using Parallel.Core.IO; -using Parallel.Core.IO.Backup; -using Parallel.Core.Models; -using Parallel.Core.Settings; -using Parallel.Core.Utils; -using Formatter = Parallel.Cli.Utils.Formatter; - -namespace Parallel.Cli.Commands -{ - public class HistoryCommand : Command - { - private const int Limit = 25; - - private Command _pullCmd = new("pull", "Shows the history related to pulling files from vaults."); - private Command _pushCmd = new("push", "Shows the history related to pushing files from vaults."); - private Command _deleteCmd = new("archive", "Shows the history related to file deletions."); - private Command _cleanCmd = new("cleaned", "Shows the history related to file cleaning."); - private Command _cloneCmd = new("cloned", "Shows the history related to file cloning."); - private Command _pruneCmd = new("pruned", "Shows the history related to file pruning."); - - private Option _sourceOpt = new(["--path", "-p"], "The source path."); - private Option _vaultOpt = new(["--vault", "-v"], "The vault to use."); - private Option _limitOpt = new(["--limit", "-l"], "The number of entries to show."); - - public HistoryCommand() : base("history", "Shows the history of files related to the archive.") - { - this.AddOption(_sourceOpt); - this.AddOption(_vaultOpt); - this.AddOption(_limitOpt); - this.AddCommand(_pullCmd); - this.AddCommand(_pushCmd); - this.AddCommand(_deleteCmd); - this.AddCommand(_cleanCmd); - this.AddCommand(_cloneCmd); - this.AddCommand(_pruneCmd); - this.SetHandler((path, config, limit) => - { - CommandLine.WriteLine($"Retrieving backup information...", ConsoleColor.DarkGray); - IDatabase? db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); - - if (limit == 0) limit = Limit; - DisplayHistories(db?.GetHistory(path, limit).ToArray()); - }, _sourceOpt, _vaultOpt, _limitOpt); - - _pushCmd.AddOption(_sourceOpt); - _pushCmd.AddOption(_vaultOpt); - _pushCmd.AddOption(_limitOpt); - _pushCmd.SetHandler((path, config, limit) => - { - CommandLine.WriteLine($"Retrieving backup information...", ConsoleColor.DarkGray); - IDatabase? db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); - - if (limit == 0) limit = Limit; - DisplayHistories(db?.GetHistory(path, HistoryType.Pushed, limit).ToArray()); - }, _sourceOpt, _vaultOpt, _limitOpt); - - _deleteCmd.AddOption(_sourceOpt); - _deleteCmd.AddOption(_vaultOpt); - _deleteCmd.AddOption(_limitOpt); - _deleteCmd.SetHandler((path, config, limit) => - { - CommandLine.WriteLine($"Retrieving archive information...", ConsoleColor.DarkGray); - IDatabase? db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); - - if (limit == 0) limit = Limit; - DisplayHistories(db?.GetHistory(path, HistoryType.Archived, limit).ToArray()); - }, _sourceOpt, _vaultOpt, _limitOpt); - - _cleanCmd.AddOption(_sourceOpt); - _cleanCmd.AddOption(_vaultOpt); - _cleanCmd.AddOption(_limitOpt); - _cleanCmd.SetHandler((path, config, limit) => - { - CommandLine.WriteLine($"Retrieving clean information...", ConsoleColor.DarkGray); - IDatabase? db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); - - if (limit == 0) limit = Limit; - DisplayHistories(db?.GetHistory(path, HistoryType.Cleaned, limit).ToArray()); - }, _sourceOpt, _vaultOpt, _limitOpt); - - _cloneCmd.AddOption(_sourceOpt); - _cloneCmd.AddOption(_vaultOpt); - _cloneCmd.AddOption(_limitOpt); - _cloneCmd.SetHandler((path, config, limit) => - { - CommandLine.WriteLine($"Retrieving clone information...", ConsoleColor.DarkGray); - IDatabase db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); - - if (limit == 0) limit = Limit; - DisplayHistories(db?.GetHistory(path, HistoryType.Cloned, limit).ToArray()); - }, _sourceOpt, _vaultOpt, _limitOpt); - - _pruneCmd.AddOption(_sourceOpt); - _pruneCmd.AddOption(_vaultOpt); - _pruneCmd.AddOption(_limitOpt); - _pruneCmd.SetHandler((path, config, limit) => - { - CommandLine.WriteLine($"Retrieving prune information...", ConsoleColor.DarkGray); - IDatabase db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); - - if (limit == 0) limit = Limit; - DisplayHistories(db?.GetHistory(path, HistoryType.Pruned, limit).ToArray()); - }, _sourceOpt, _vaultOpt, _limitOpt); - - _pullCmd.AddOption(_sourceOpt); - _pullCmd.AddOption(_vaultOpt); - _pullCmd.AddOption(_limitOpt); - _pullCmd.SetHandler((path, config, limit) => - { - CommandLine.WriteLine($"Retrieving restore information...", ConsoleColor.DarkGray); - IDatabase db = DatabaseConnection.CreateNew(RemoteVaultConfig.Load(Program.Settings, config)); - - if (limit == 0) limit = Limit; - DisplayHistories(db?.GetHistory(path, HistoryType.Pulled, limit).ToArray()); - }, _sourceOpt, _vaultOpt, _limitOpt); - } - - private void DisplayHistories(HistoryEvent[]? histories) - { - if (histories?.Length == 0) - { - CommandLine.WriteLine("No backup history found!", ConsoleColor.Yellow); - return; - } - - foreach (HistoryEvent history in histories.ToArray()) - { - string typeStr = (history.Type + ":").PadRight(9); - CommandLine.WriteLine($"[{Formatter.FromDateTime(history.CreatedAt.ToLocalTime())}] <{history.Vault}> {typeStr} {history.Fullname}", ConsoleColor.White); - } - } - } -} \ No newline at end of file diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 49e3bfc..97b869d 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -47,10 +47,10 @@ private async Task SyncSystemAsync() private async Task SyncPathAsync(string path) { - await ParallelConfig.ForEachVaultAsync(async vault => + await Program.Settings.ForEachVaultAsync(async vault => { - ISyncManager sync = SyncManager.CreateNew(vault); - if (!await sync.InitializeAsync()) + FileSyncManager syncManager = new FileSyncManager(vault); + if (!await syncManager.ConnectAsync()) { CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); return; @@ -58,8 +58,8 @@ await ParallelConfig.ForEachVaultAsync(async vault => // Normalize paths for safe comparison string fullPath = Path.GetFullPath(path); - string[] backupFolders = vault.BackupDirectories.ToArray(); - string[] ignoredFolders = vault.IgnoreDirectories.ToArray(); + string[] backupFolders = syncManager.RemoteVault.BackupDirectories.ToArray(); + string[] ignoredFolders = syncManager.RemoteVault.IgnoreDirectories.ToArray(); bool isFile = PathBuilder.IsFile(fullPath); if (!backupFolders.Any(dir => fullPath.StartsWith(dir, StringComparison.OrdinalIgnoreCase))) @@ -75,8 +75,7 @@ await ParallelConfig.ForEachVaultAsync(async vault => } CommandLine.WriteLine(vault, $"Scanning for file changes in {path}...", ConsoleColor.DarkGray); - - FileScanner scanner = new FileScanner(sync); + FileScanner scanner = new FileScanner(syncManager); SystemFile[] files = await scanner.GetFileChangesAsync(path, ignoredFolders); int successFiles = files.Length; if (successFiles == 0) @@ -86,10 +85,10 @@ await ParallelConfig.ForEachVaultAsync(async vault => } CommandLine.WriteLine(vault, $"Backing up {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); - await sync.PushFilesAsync(files, new ProgressReport(vault)); + await syncManager.PushFilesAsync(files, new ProgressReport(vault, successFiles)); + await syncManager.DisconnectAsync(); CommandLine.WriteLine(vault, $"Successfully pushed {successFiles.ToString("N0")} files to '{vault.FileSystem.Address}'.", ConsoleColor.Green); - await sync.DisconnectAsync(); }); } } diff --git a/Parallel.Cli/Program.cs b/Parallel.Cli/Program.cs index 7369bd9..b5b3c43 100644 --- a/Parallel.Cli/Program.cs +++ b/Parallel.Cli/Program.cs @@ -14,7 +14,10 @@ internal class Program public static async Task Main(string[] args) { Settings = ParallelConfig.Load(); - string logFile = Path.Combine(PathBuilder.ProgramData, "Logs", $"{DateTime.Now:MM-dd-yyyy hh-mm-ss}.log"); + //string logFile = Path.Combine(PathBuilder.ProgramData, "Logs", $"{DateTime.Now:MM-dd-yyyy hh-mm-ss}.log"); + + string logFile = Path.Combine(PathBuilder.ProgramData, "Logs", "latest.txt"); + if (File.Exists(logFile)) File.Delete(logFile); Log.Logger = new LoggerConfiguration().MinimumLevel.Debug().WriteTo.File(logFile).CreateLogger(); AssemblyName assembly = Assembly.GetExecutingAssembly().GetName(); diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index aaf6525..c691094 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -22,9 +22,9 @@ public class SqliteContext : IDatabase /// /// /// - public SqliteContext(LocalVaultConfig localVault) + public SqliteContext(string filePath) { - FilePath = PathBuilder.GetDatabaseFile(localVault); + FilePath = filePath; } #region Base @@ -40,7 +40,7 @@ public async Task InitializeAsync() { Log.Information("Creating index database..."); File.Create(FilePath).Close(); - File.SetAttributes(FilePath, File.GetAttributes(FilePath) | FileAttributes.Hidden); + //File.SetAttributes(FilePath, File.GetAttributes(FilePath) | FileAttributes.Hidden); using IDbConnection connection = CreateConnection(); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` TEXT, PRIMARY KEY(`id`));"); diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index d608956..fea83db 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -57,18 +57,16 @@ public Task DeleteFileAsync(string path) return Task.CompletedTask; } - public Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress) - { - throw new NotImplementedException(); - } - /// - public async Task DownloadFileAsync(string sourcePath, string destPath) + public async Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress) { - await using FileStream openStream = File.OpenRead(sourcePath); - await using FileStream createStream = File.Create(destPath); - await using GZipStream gzipStream = new GZipStream(openStream, CompressionMode.Decompress); - await gzipStream.CopyToAsync(createStream); + await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + { + await using FileStream openStream = File.OpenRead(file.RemotePath); + await using FileStream createStream = File.Create(file.LocalPath); + await using GZipStream gzipStream = new GZipStream(openStream, CompressionMode.Decompress); + await gzipStream.CopyToAsync(createStream, ct); + }); } /// @@ -89,29 +87,22 @@ public Task GetFileAsync(string path) }); } + /// public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress) { - await System.Threading.Tasks.Parallel.ForEachAsync(files, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount / 2 }, async (file, ct) => + await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { + if (await ExistsAsync(file.RemotePath)) File.SetAttributes(file.RemotePath, ~FileAttributes.ReadOnly & File.GetAttributes(file.RemotePath)); + string? parent = Path.GetDirectoryName(file.RemotePath); + if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); - }); - } + await using FileStream openStream = File.OpenRead(file.LocalPath); + await using FileStream createStream = File.Create(file.RemotePath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await openStream.CopyToAsync(gzipStream, ct); - /// - public async Task UploadFileAsync(string sourcePath, string destPath) - { - - - if (await ExistsAsync(destPath)) File.SetAttributes(destPath, ~FileAttributes.ReadOnly & File.GetAttributes(destPath)); - string? parent = Path.GetDirectoryName(destPath); - if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); - - await using FileStream openStream = File.OpenRead(sourcePath); - await using FileStream createStream = File.Create(destPath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream); - - File.SetAttributes(destPath, File.GetAttributes(destPath) | FileAttributes.ReadOnly); + File.SetAttributes(file.RemotePath, File.GetAttributes(file.RemotePath) | FileAttributes.ReadOnly); + }); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index db16c8a..4cf7b6d 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -22,6 +22,7 @@ public static string TempDirectory get { string tempFolder = Path.Combine(Path.GetTempPath(), "Parallel"); + Log.Debug($"Temp directory: {tempFolder}"); if (!Directory.Exists(tempFolder)) Directory.CreateDirectory(tempFolder); return tempFolder; } @@ -127,7 +128,7 @@ public static string GetSnapshotsDirectory(LocalVaultConfig localVault) /// public static string GetConfigurationFile(LocalVaultConfig localVault) { - return Combine(GetRootDirectory(localVault), "config.json"); + return Combine(GetRootDirectory(localVault), "config.json.gz"); } /// @@ -137,7 +138,7 @@ public static string GetConfigurationFile(LocalVaultConfig localVault) /// public static string GetDatabaseFile(LocalVaultConfig localVault) { - return Combine(GetRootDirectory(localVault), "index.db"); + return Combine(GetRootDirectory(localVault), "index.db.gz"); } /// diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index d39ea53..264c1fe 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -39,37 +39,11 @@ public BaseSyncManager(LocalVaultConfig localVault) LocalVault = localVault; } - /// - public void Dispose() - { - FileSystem.Dispose(); - } - /// public async Task InitializeAsync() { try { - string root = PathBuilder.GetRootDirectory(LocalVault); - if (!await FileSystem.ExistsAsync(root)) - { - // Creates the root directory and default configuration. - await FileSystem.CreateDirectoryAsync(root); - RemoteVault = new RemoteVaultConfig(LocalVault); - } - else - { - SystemFile[] files = - [ - new SystemFile(TempConfigFile) { RemotePath = PathBuilder.GetConfigurationFile(LocalVault) }, - new SystemFile(TempDbFile) { RemotePath = PathBuilder.GetDatabaseFile(LocalVault) }, - ]; - - await FileSystem.DownloadFilesAsync(files, new ProgressLogger()); - } - - Database = new SqliteContext(LocalVault); - RemoteVault.IgnoreDirectories.Add(PathBuilder.GetRootDirectory(LocalVault)); return true; } catch (Exception ex) @@ -79,11 +53,48 @@ public async Task InitializeAsync() } } + public async Task ConnectAsync() + { + string root = PathBuilder.GetRootDirectory(LocalVault); + if (!await FileSystem.ExistsAsync(root)) + { + await FileSystem.CreateDirectoryAsync(root); + Log.Debug($"Created root directory: {root}"); + } + + if (!await FileSystem.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) + { + RemoteVault = new RemoteVaultConfig(LocalVault); + RemoteVault.IgnoreDirectories.Add(PathBuilder.GetRootDirectory(LocalVault)); + RemoteVault.Save(TempConfigFile); + + Log.Debug($"Created config file: {TempConfigFile}"); + } + else + { + await FileSystem.DownloadFilesAsync([new SystemFile { LocalPath = TempConfigFile, RemotePath = PathBuilder.GetConfigurationFile(LocalVault) }], new ProgressLogger()); + RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); + if(config == null) return false; + RemoteVault = config; + } + + if (!await FileSystem.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) + { + Database = new SqliteContext(TempDbFile); + await Database.InitializeAsync(); + } + else + { + await FileSystem.DownloadFilesAsync([new SystemFile { LocalPath = TempDbFile, RemotePath = PathBuilder.GetDatabaseFile(LocalVault) }], new ProgressLogger()); + Database = new SqliteContext(TempDbFile); + } + + return true; + } + /// public Task DisconnectAsync() { - string configFile = PathBuilder.GetConfigurationFile(LocalVault); - FileSystem.Dispose(); return Task.CompletedTask; } diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 66cfe71..3e6304a 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -16,6 +16,9 @@ namespace Parallel.Core.IO.Backup /// public class FileSyncManager : BaseSyncManager { + protected string[] BackupDirectories => RemoteVault.BackupDirectories.ToArray(); + protected string[] IgnoreDirectories => RemoteVault.IgnoreDirectories.ToArray(); + /// /// Initializes a new instance of the class. /// diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index 9092bce..15d1395 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -11,7 +11,7 @@ namespace Parallel.Core.IO.Syncing /// /// Defines the methods needed for backing up a file system. /// - public interface ISyncManager : IDisposable + public interface ISyncManager { /// /// Gets the local vault configuration. @@ -34,9 +34,14 @@ public interface ISyncManager : IDisposable IFileSystem FileSystem { get; set; } /// - /// Initializes the associated and downloads the needed files. + /// Establishes a connection to the associated and downloads the needed files. /// - Task InitializeAsync(); + Task ConnectAsync(); + + /// + /// Closes the current connection and releases its resources. + /// + Task DisconnectAsync(); /// /// Pushes an array of files to a vault. diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index 5e2a144..a98dab6 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -156,6 +156,8 @@ public SystemFile(string vault, string id, string name, string localpath, string CheckSum = checksum; } + public SystemFile() { } + /// /// Determines if this instance and another have the same values. /// diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs index a5a405d..9d9ef39 100644 --- a/Parallel.Core/Settings/LocalVaultConfig.cs +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -40,7 +40,7 @@ public LocalVaultConfig(string id, string name, FileSystemCredentials fileSystem public LocalVaultConfig(string name, FileSystemCredentials fileSystem) { - Id = HashGenerator.GenerateHash(12, true); + Id = HashGenerator.GenerateHash(8, true); Name = name; FileSystem = fileSystem; } diff --git a/Parallel.Core/Settings/ParallelConfig.cs b/Parallel.Core/Settings/ParallelConfig.cs index 2083801..ada6eb7 100644 --- a/Parallel.Core/Settings/ParallelConfig.cs +++ b/Parallel.Core/Settings/ParallelConfig.cs @@ -21,6 +21,11 @@ public class ParallelConfig /// public static string VaultsDir { get; } = Path.Combine(PathBuilder.ProgramData, "Vaults"); + public static ParallelOptions Options { get; } = new ParallelOptions + { + MaxDegreeOfParallelism = Load().MaxConcurrentProcesses + }; + // /// // /// The address that will accept incoming commands. // /// Default: 127.0.0.1 @@ -43,7 +48,7 @@ public class ParallelConfig /// Gets or sets the maximum number of concurrent processes that can run. /// Default: Half the processor count. /// - public int MaxConcurrentProcesses { get; set; } = Environment.ProcessorCount / 2; + public int MaxConcurrentProcesses { get; set; } = Math.Clamp(Environment.ProcessorCount / 2, 1, Environment.ProcessorCount); /// /// The profiles to use. diff --git a/Parallel.Core/Settings/RemoteVaultConfig.cs b/Parallel.Core/Settings/RemoteVaultConfig.cs index a9dab5a..e885429 100644 --- a/Parallel.Core/Settings/RemoteVaultConfig.cs +++ b/Parallel.Core/Settings/RemoteVaultConfig.cs @@ -60,10 +60,23 @@ public class RemoteVaultConfig : LocalVaultConfig /// public HashSet PruneDirectories { get; } = []; + public RemoteVaultConfig(LocalVaultConfig localVault) : base(localVault.Id, localVault.Name, localVault.FileSystem) { } public RemoteVaultConfig(string profileName, FileSystemCredentials fsc) : base(profileName, fsc) { } + [JsonConstructor] + public RemoteVaultConfig(string id, string name, FileSystemCredentials fileSystem, int backupInterval, int retentionPeriod, int prunePeriod, IEnumerable backupDirectories, IEnumerable ignoreDirectories, IEnumerable cleanDirectories, IEnumerable pruneDirectories) : base(id, name, fileSystem) + { + BackupInterval = backupInterval; + RetentionPeriod = retentionPeriod; + PrunePeriod = prunePeriod; + BackupDirectories = new HashSet(backupDirectories); + IgnoreDirectories = new HashSet(ignoreDirectories); + CleanDirectories = new HashSet(cleanDirectories); + PruneDirectories = new HashSet(pruneDirectories); + } + #region Privates private static HashSet CreateBackupDirectories() @@ -116,5 +129,18 @@ private static HashSet CreateCleanDirectories() } #endregion + + /// + /// Loads settings from a file. + /// + public new static RemoteVaultConfig? Load(string path) + { + return !File.Exists(path) ? null : JsonConvert.DeserializeObject(File.ReadAllText(path)); + } + + public void Save(string path) + { + File.WriteAllText(path, JsonConvert.SerializeObject(this, Formatting.Indented)); + } } } \ No newline at end of file From f29df6ae29a870c339eda3b26abda8bfdd4ba6fe Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 2 Sep 2025 03:00:50 -0500 Subject: [PATCH 3/8] Now that databases are instanced there is no need to store a vault id --- .../Database/Contexts/SqliteContext.cs | 17 +++++++++-------- Parallel.Core/Database/IDatabase.cs | 5 ----- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index c691094..8ebcac2 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -14,8 +14,7 @@ namespace Parallel.Core.Database /// public class SqliteContext : IDatabase { - public string FilePath { get; } - public string ProfileId { get; } + private string FilePath { get; } /// /// Initializes a new instance of the class. @@ -55,15 +54,15 @@ public async Task InitializeAsync() public async Task AddFileAsync(SystemFile file) { using IDbConnection connection = CreateConnection(); - string sql = @"INSERT OR REPLACE INTO files (vault, id, name, localpath, remotepath, lastwrite, lastupdate, LocalSize, RemoteSize, type, hidden, readonly, deleted, encrypted, salt, iv, checksum) VALUES (@ProfileId, @Id, @Name, @LocalPath, @RemotePath, @LastWrite, @LastUpdate, @LocalSize, @RemoteSize, @Type, @Hidden, @ReadOnly, @Deleted, @Encrypted, @Salt, @IV, @CheckSum);"; - return await connection.ExecuteAsync(sql, new { ProfileId, file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.Encrypted, file.Salt, file.IV, file.CheckSum }) > 0; + string sql = @"INSERT OR REPLACE INTO files (id, name, localpath, remotepath, lastwrite, lastupdate, LocalSize, RemoteSize, type, hidden, readonly, deleted, encrypted, salt, iv, checksum) VALUES (@ProfileId, @Id, @Name, @LocalPath, @RemotePath, @LastWrite, @LastUpdate, @LocalSize, @RemoteSize, @Type, @Hidden, @ReadOnly, @Deleted, @Encrypted, @Salt, @IV, @CheckSum);"; + return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.Encrypted, file.Salt, file.IV, file.CheckSum }) > 0; } /// public async Task> GetFilesAsync(string path, bool deleted) { using IDbConnection connection = CreateConnection(); - string sql = $"SELECT * FROM files WHERE vault = \"{ProfileId}\" AND deleted = {deleted} ORDER BY lastupdate DESC"; + string sql = $"SELECT * FROM files WHERE deleted = {deleted} ORDER BY lastupdate DESC"; return await connection.QueryAsync(sql); } @@ -71,7 +70,7 @@ public async Task> GetFilesAsync(string path, bool delet public async Task GetFileAsync(string path) { using IDbConnection connection = CreateConnection(); - string sql = $"SELECT * FROM files WHERE vault = \"{ProfileId}\" AND localpath LIKE \"%{path}%\" OR remotepath LIKE \"%{path}%\" ORDER BY lastupdate DESC"; + string sql = $"SELECT * FROM files WHERE localpath LIKE \"%{path}%\" OR remotepath LIKE \"%{path}%\" ORDER BY lastupdate DESC"; return await connection.QuerySingleOrDefaultAsync(sql); } @@ -83,15 +82,17 @@ public async Task> GetFilesAsync(string path, bool delet public async Task AddHistoryAsync(string path, HistoryType type) { using IDbConnection connection = CreateConnection(); - string sql = @"INSERT OR REPLACE INTO history (vault, timestamp, name, path, type) VALUES(@ProfileId, @Timestamp, @Name, @Path, @Type);"; - return await connection.ExecuteAsync(sql, new { ProfileId, Timestamp = UnixTime.Now.TotalMilliseconds, Name = Path.GetFileName(path), Path = path, Type = type }) > 0; + string sql = @"INSERT OR REPLACE INTO history (timestamp, name, path, type) VALUES(@ProfileId, @Timestamp, @Name, @Path, @Type);"; + return await connection.ExecuteAsync(sql, new { Timestamp = UnixTime.Now.TotalMilliseconds, Name = Path.GetFileName(path), Path = path, Type = type }) > 0; } + /// public IEnumerable? GetHistory(string path, int limit) { throw new NotImplementedException(); } + /// public IEnumerable? GetHistory(string path, HistoryType type, int limit) { throw new NotImplementedException(); diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 65fea25..2b2c055 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -49,11 +49,6 @@ public enum HistoryType /// public interface IDatabase { - /// - /// The identifier to the vault for this database. - /// - string ProfileId { get; } - #region Base /// From 8f74ee50266a76daea57769377dcfe71ab16a89b Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 2 Sep 2025 04:16:17 -0500 Subject: [PATCH 4/8] Works fine except for uploading the database --- Parallel.Cli/Commands/PushCommand.cs | 1 + Parallel.Cli/Utils/ProgressReport.cs | 1 + .../Database/Contexts/SqliteContext.cs | 40 +++++++++++++------ Parallel.Core/Database/IDatabase.cs | 2 +- .../IO/FileSystem/DotNetFileSystem.cs | 23 +++++++---- Parallel.Core/IO/Scanning/FileScanner.cs | 15 ++++--- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 14 +++++-- Parallel.Core/IO/Syncing/FileSyncManager.cs | 23 +++-------- Parallel.Core/Models/SystemFile.cs | 35 ++++------------ 9 files changed, 74 insertions(+), 80 deletions(-) diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 97b869d..d618d16 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -81,6 +81,7 @@ await Program.Settings.ForEachVaultAsync(async vault => if (successFiles == 0) { CommandLine.WriteLine(vault, $"The provided {(isFile ? "file" : "folder")} is already up to date.", ConsoleColor.Green); + await syncManager.DisconnectAsync(); return; } diff --git a/Parallel.Cli/Utils/ProgressReport.cs b/Parallel.Cli/Utils/ProgressReport.cs index f2c2f9b..4f2df48 100644 --- a/Parallel.Cli/Utils/ProgressReport.cs +++ b/Parallel.Cli/Utils/ProgressReport.cs @@ -26,6 +26,7 @@ public void Report(ProgressOperation operation, SystemFile file, int current, in public void Failed(Exception exception, SystemFile file) { CommandLine.WriteLine(_localVault, $"Failed to upload file: '{file.LocalPath}'", ConsoleColor.Red); + Log.Error(exception.GetBaseException().ToString()); } } } \ No newline at end of file diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 8ebcac2..70c1e6d 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -26,6 +26,11 @@ public SqliteContext(string filePath) FilePath = filePath; } + public void Dispose() + { + // TODO release managed resources here + } + #region Base /// @@ -38,10 +43,11 @@ public IDbConnection CreateConnection() public async Task InitializeAsync() { Log.Information("Creating index database..."); - File.Create(FilePath).Close(); + //File.Create(FilePath).Close(); //File.SetAttributes(FilePath, File.GetAttributes(FilePath) | FileAttributes.Hidden); using IDbConnection connection = CreateConnection(); + await connection.ExecuteAsync("PRAGMA journal_mode=WAL;"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` TEXT, PRIMARY KEY(`id`));"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `history` (`timestamp` LONG INTEGER NOT NULL, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`timestamp`));"); } @@ -53,25 +59,31 @@ public async Task InitializeAsync() /// public async Task AddFileAsync(SystemFile file) { - using IDbConnection connection = CreateConnection(); - string sql = @"INSERT OR REPLACE INTO files (id, name, localpath, remotepath, lastwrite, lastupdate, LocalSize, RemoteSize, type, hidden, readonly, deleted, encrypted, salt, iv, checksum) VALUES (@ProfileId, @Id, @Name, @LocalPath, @RemotePath, @LastWrite, @LastUpdate, @LocalSize, @RemoteSize, @Type, @Hidden, @ReadOnly, @Deleted, @Encrypted, @Salt, @IV, @CheckSum);"; - return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.Encrypted, file.Salt, file.IV, file.CheckSum }) > 0; + using (IDbConnection connection = CreateConnection()) + { + string sql = @"INSERT OR REPLACE INTO files (id, name, localpath, remotepath, lastwrite, lastupdate, LocalSize, RemoteSize, type, hidden, readonly, deleted, checksum) VALUES (@Id, @Name, @LocalPath, @RemotePath, @LastWrite, @LastUpdate, @LocalSize, @RemoteSize, @Type, @Hidden, @ReadOnly, @Deleted, @CheckSum);"; + return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.CheckSum }) > 0; + } } /// public async Task> GetFilesAsync(string path, bool deleted) { - using IDbConnection connection = CreateConnection(); - string sql = $"SELECT * FROM files WHERE deleted = {deleted} ORDER BY lastupdate DESC"; - return await connection.QueryAsync(sql); + using (IDbConnection connection = CreateConnection()) + { + string sql = $"SELECT * FROM files WHERE deleted = {deleted} ORDER BY lastupdate DESC"; + return await connection.QueryAsync(sql); + } } /// public async Task GetFileAsync(string path) { - using IDbConnection connection = CreateConnection(); - string sql = $"SELECT * FROM files WHERE localpath LIKE \"%{path}%\" OR remotepath LIKE \"%{path}%\" ORDER BY lastupdate DESC"; - return await connection.QuerySingleOrDefaultAsync(sql); + using (IDbConnection connection = CreateConnection()) + { + string sql = $"SELECT (id, name, localpath, remotepath, lastwrite, lastupdate, LocalSize, RemoteSize, type, hidden, readonly, deleted, checksum) FROM files WHERE localpath LIKE \"%{path}%\" OR remotepath LIKE \"%{path}%\" ORDER BY lastupdate DESC"; + return await connection.QuerySingleOrDefaultAsync(sql); + } } #endregion @@ -81,9 +93,11 @@ public async Task> GetFilesAsync(string path, bool delet /// public async Task AddHistoryAsync(string path, HistoryType type) { - using IDbConnection connection = CreateConnection(); - string sql = @"INSERT OR REPLACE INTO history (timestamp, name, path, type) VALUES(@ProfileId, @Timestamp, @Name, @Path, @Type);"; - return await connection.ExecuteAsync(sql, new { Timestamp = UnixTime.Now.TotalMilliseconds, Name = Path.GetFileName(path), Path = path, Type = type }) > 0; + using (IDbConnection connection = CreateConnection()) + { + string sql = @"INSERT OR REPLACE INTO history (timestamp, name, path, type) VALUES(@Timestamp, @Name, @Path, @Type);"; + return await connection.ExecuteAsync(sql, new { Timestamp = UnixTime.Now.TotalMilliseconds, Name = Path.GetFileName(path), Path = path, Type = type }) > 0; + } } /// diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 2b2c055..698be33 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -47,7 +47,7 @@ public enum HistoryType /// /// An interface for interacting with client data storage. /// - public interface IDatabase + public interface IDatabase : IDisposable { #region Base diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index fea83db..d095a33 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -92,16 +92,23 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres { await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - if (await ExistsAsync(file.RemotePath)) File.SetAttributes(file.RemotePath, ~FileAttributes.ReadOnly & File.GetAttributes(file.RemotePath)); - string? parent = Path.GetDirectoryName(file.RemotePath); - if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); + try + { + if (await ExistsAsync(file.RemotePath)) File.SetAttributes(file.RemotePath, ~FileAttributes.ReadOnly & File.GetAttributes(file.RemotePath)); + string? parent = Path.GetDirectoryName(file.RemotePath); + if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); - await using FileStream openStream = File.OpenRead(file.LocalPath); - await using FileStream createStream = File.Create(file.RemotePath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream, ct); + await using FileStream openStream = File.OpenRead(file.LocalPath); + await using FileStream createStream = File.Create(file.RemotePath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await openStream.CopyToAsync(gzipStream, ct); - File.SetAttributes(file.RemotePath, File.GetAttributes(file.RemotePath) | FileAttributes.ReadOnly); + File.SetAttributes(file.RemotePath, File.GetAttributes(file.RemotePath) | FileAttributes.ReadOnly); + } + catch (Exception ex) + { + progress.Failed(ex, file); + } }); } } diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index aea7a6b..993ce9f 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -19,16 +19,13 @@ namespace Parallel.Core.IO.Scanning /// public class FileScanner { + private readonly RemoteVaultConfig _config; private readonly IDatabase _db; - public FileScanner(IDatabase database) + public FileScanner(ISyncManager syncManager) { - _db = database; - } - - public FileScanner(ISyncManager sync) - { - _db = sync.Database; + _config = syncManager.RemoteVault; + _db = syncManager.Database; } /*/// @@ -69,12 +66,14 @@ public async Task GetFileChangesAsync(string path, string[] ignore if (IsIgnored(localFile.LocalPath, ignoreFolders)) { Log.Debug($"Ignored -> {localFile.LocalPath}"); + localFile.RemotePath = remoteFile.RemotePath; localFile.Deleted = true; scannedFiles.Add(localFile); } else if (HasChanged(localFile, remoteFile)) { Log.Debug($"Changed -> {localFile.LocalPath}"); + localFile.RemotePath = remoteFile.RemotePath; scannedFiles.Add(localFile); } @@ -94,7 +93,7 @@ public async Task GetFileChangesAsync(string path, string[] ignore if (File.Exists(file) && !IsIgnored(file, ignoreFolders)) { Log.Debug($"Created -> {file}"); - scannedFiles.Add(new SystemFile(file)); + scannedFiles.Add(new SystemFile(file) { RemotePath = PathBuilder.Remote(file, _config) }); } } diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index 264c1fe..c9745c4 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -72,31 +72,37 @@ public async Task ConnectAsync() } else { - await FileSystem.DownloadFilesAsync([new SystemFile { LocalPath = TempConfigFile, RemotePath = PathBuilder.GetConfigurationFile(LocalVault) }], new ProgressLogger()); + await FileSystem.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new ProgressLogger()); RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); if(config == null) return false; RemoteVault = config; + + Log.Debug($"Downloaded config file: {TempConfigFile}"); } if (!await FileSystem.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) { Database = new SqliteContext(TempDbFile); await Database.InitializeAsync(); + + Log.Debug($"Create db file: {TempDbFile}"); } else { - await FileSystem.DownloadFilesAsync([new SystemFile { LocalPath = TempDbFile, RemotePath = PathBuilder.GetDatabaseFile(LocalVault) }], new ProgressLogger()); + await FileSystem.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new ProgressLogger()); Database = new SqliteContext(TempDbFile); + + Log.Debug($"Downloaded db file: {TempDbFile}"); } return true; } /// - public Task DisconnectAsync() + public async Task DisconnectAsync() { + await FileSystem.UploadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault)), new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new ProgressLogger()); FileSystem.Dispose(); - return Task.CompletedTask; } /// diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 3e6304a..a1f388e 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -16,17 +16,11 @@ namespace Parallel.Core.IO.Backup /// public class FileSyncManager : BaseSyncManager { - protected string[] BackupDirectories => RemoteVault.BackupDirectories.ToArray(); - protected string[] IgnoreDirectories => RemoteVault.IgnoreDirectories.ToArray(); - /// /// Initializes a new instance of the class. /// /// - public FileSyncManager(LocalVaultConfig localVault) : base(localVault) - { - - } + public FileSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) @@ -35,24 +29,17 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter SystemFile[] backupFiles = files.Where(f => !f.Deleted).ToArray(); Log.Information($"Backing up {backupFiles.Length} files..."); await FileSystem.UploadFilesAsync(backupFiles, progress); - await System.Threading.Tasks.Parallel.ForEachAsync(files, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount / 2 }, async (file, ct) => - { - - }); - - - for (int i = 0; i < files.Length; i++) + await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - SystemFile file = files.ElementAt(i); if (file.Deleted) { - progress.Report(ProgressOperation.Archiving, file, i, files.Length); + progress.Report(ProgressOperation.Archiving, file, 0, files.Length); await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); await Database.AddFileAsync(file); } else { - progress.Report(ProgressOperation.Syncing, file, i, files.Length); + progress.Report(ProgressOperation.Syncing, file, 0, files.Length); SystemFile remote = await FileSystem.GetFileAsync(file.RemotePath); if (remote is not null) { @@ -61,7 +48,7 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter await Database.AddFileAsync(file); } } - } + }); } /// diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index a98dab6..2b9f6a9 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -73,21 +73,6 @@ public class SystemFile /// public bool Deleted { get; set; } = false; - /// - /// If the file is encrypted in the backup. - /// - public bool Encrypted { get; set; } = false; - - /// - /// The salt used to encrypt the file. - /// - public string Salt { get; set; } - - /// - /// The initialization vector used to encrypt the file. - /// - public string IV { get; set; } - /// /// The checksum used to check if the file has changed. /// @@ -111,16 +96,18 @@ public SystemFile(string path) Hidden = fileInfo.Attributes.HasFlag(FileAttributes.Hidden); ReadOnly = fileInfo.Attributes.HasFlag(FileAttributes.ReadOnly); Deleted = !fileInfo.Exists; - Encrypted = false; - Salt = HashGenerator.GenerateHash(16); - IV = HashGenerator.GenerateHash(16); CheckSum = HashGenerator.CheckSum(path); } + public SystemFile(string localPath, string remotePath) + { + LocalPath = localPath; + RemotePath = remotePath; + } + /// /// Initializes a new instance of the class. /// - /// /// /// /// @@ -137,7 +124,7 @@ public SystemFile(string path) /// /// /// - public SystemFile(string vault, string id, string name, string localpath, string remotepath, long lastwrite, long lastupdate, long localsize, long remotesize, string type, long hidden, long readOnly, long deleted, long encrypted, string salt, string iv, string checksum) + public SystemFile(string id, string name, string localpath, string remotepath, long lastwrite, long lastupdate, long localsize, long remotesize, string type, long hidden, long readOnly, long deleted, string checksum) { Id = id; Name = name; @@ -150,14 +137,9 @@ public SystemFile(string vault, string id, string name, string localpath, string Hidden = Converter.ToBool(hidden); ReadOnly = Converter.ToBool(readOnly); Deleted = Converter.ToBool(deleted); - Encrypted = Converter.ToBool(encrypted); - Salt = salt; - IV = iv; CheckSum = checksum; } - public SystemFile() { } - /// /// Determines if this instance and another have the same values. /// @@ -177,9 +159,6 @@ public bool Equals(SystemFile value) value?.Hidden != null ? this.Hidden.Equals(value.Hidden) : (bool?)null, value?.ReadOnly != null ? this.ReadOnly.Equals(value.ReadOnly) : (bool?)null, value?.Deleted != null ? this.Deleted.Equals(value.Deleted) : (bool?)null, - value?.Encrypted != null ? this.Encrypted.Equals(value.Encrypted) : (bool?)null, - this?.Salt != null && value?.Salt != null ? this.Salt.SequenceEqual(value.Salt) : (bool?)null, - this?.IV != null && value?.IV != null ? this.IV.SequenceEqual(value.IV) : (bool?)null, this?.CheckSum != null && value?.CheckSum != null ? this.CheckSum.SequenceEqual(value.CheckSum) : (bool?)null, ]; From bf7832bb90269e5aedc39fb3ff429c7ea03cbd16 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 2 Sep 2025 18:23:50 -0500 Subject: [PATCH 5/8] Successfully pushes files --- Parallel.Cli/Commands/PushCommand.cs | 3 +- Parallel.Cli/Utils/ProgressReport.cs | 17 +++++++--- .../Database/Contexts/SqliteContext.cs | 5 +-- .../Diagnostics/IProgressReporter.cs | 7 +++- Parallel.Core/Diagnostics/ProgressLogger.cs | 24 ++++++++----- .../IO/FileSystem/DotNetFileSystem.cs | 34 ++++++++++++++++++- Parallel.Core/IO/FileSystem/IFileSystem.cs | 14 ++++++++ Parallel.Core/IO/FileSystem/SftpFileSystem.cs | 32 ++++++++++++++++- Parallel.Core/IO/Syncing/FileSyncManager.cs | 6 ++-- 9 files changed, 119 insertions(+), 23 deletions(-) diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index d618d16..1b6204e 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -2,6 +2,7 @@ using System.CommandLine; using Parallel.Cli.Utils; +using Parallel.Core.Diagnostics; using Parallel.Core.IO; using Parallel.Core.IO.Backup; using Parallel.Core.IO.Scanning; @@ -89,7 +90,7 @@ await Program.Settings.ForEachVaultAsync(async vault => await syncManager.PushFilesAsync(files, new ProgressReport(vault, successFiles)); await syncManager.DisconnectAsync(); - CommandLine.WriteLine(vault, $"Successfully pushed {successFiles.ToString("N0")} files to '{vault.FileSystem.Address}'.", ConsoleColor.Green); + CommandLine.WriteLine(vault, $"Successfully pushed {successFiles.ToString("N0")} files to '{vault.FileSystem.RootDirectory}'.", ConsoleColor.Green); }); } } diff --git a/Parallel.Cli/Utils/ProgressReport.cs b/Parallel.Cli/Utils/ProgressReport.cs index 4f2df48..e9c587c 100644 --- a/Parallel.Cli/Utils/ProgressReport.cs +++ b/Parallel.Cli/Utils/ProgressReport.cs @@ -9,24 +9,31 @@ namespace Parallel.Cli.Utils public class ProgressReport : IProgressReporter { private readonly LocalVaultConfig _localVault; - private readonly int _totalFiles; + private int _current; + private int _total; public ProgressReport(LocalVaultConfig localVault, int totalFiles) { _localVault = localVault; - _totalFiles = totalFiles; + _current = 0; + _total = totalFiles; } - public void Report(ProgressOperation operation, SystemFile file, int current, int total) + public void Report(ProgressOperation operation, SystemFile file) { - int percent = current * 100 / total; + int percent = _current++ * 100 / _total; CommandLine.WriteLine($"[{percent}%] <{_localVault.Id}> {operation}: {file.LocalPath}"); } + /// + public void Reset() + { + _current = 0; + } + public void Failed(Exception exception, SystemFile file) { CommandLine.WriteLine(_localVault, $"Failed to upload file: '{file.LocalPath}'", ConsoleColor.Red); - Log.Error(exception.GetBaseException().ToString()); } } } \ No newline at end of file diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 70c1e6d..f4ca34d 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -36,18 +36,15 @@ public void Dispose() /// public IDbConnection CreateConnection() { - return new SqliteConnection("Data Source=" + FilePath); + return new SqliteConnection($"Data Source={FilePath};Pooling=false;"); } /// public async Task InitializeAsync() { Log.Information("Creating index database..."); - //File.Create(FilePath).Close(); - //File.SetAttributes(FilePath, File.GetAttributes(FilePath) | FileAttributes.Hidden); using IDbConnection connection = CreateConnection(); - await connection.ExecuteAsync("PRAGMA journal_mode=WAL;"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` TEXT, PRIMARY KEY(`id`));"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `history` (`timestamp` LONG INTEGER NOT NULL, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`timestamp`));"); } diff --git a/Parallel.Core/Diagnostics/IProgressReporter.cs b/Parallel.Core/Diagnostics/IProgressReporter.cs index a774bd1..943e0da 100644 --- a/Parallel.Core/Diagnostics/IProgressReporter.cs +++ b/Parallel.Core/Diagnostics/IProgressReporter.cs @@ -22,7 +22,12 @@ public interface IProgressReporter /// /// Reports a progress update. /// - void Report(ProgressOperation operation, SystemFile file, int current, int total); + void Report(ProgressOperation operation, SystemFile file); + + /// + /// Resets the ticking. + /// + void Reset(); /// /// Reports a failed update. diff --git a/Parallel.Core/Diagnostics/ProgressLogger.cs b/Parallel.Core/Diagnostics/ProgressLogger.cs index c624e39..8dd8463 100644 --- a/Parallel.Core/Diagnostics/ProgressLogger.cs +++ b/Parallel.Core/Diagnostics/ProgressLogger.cs @@ -10,27 +10,35 @@ namespace Parallel.Core.Diagnostics public class ProgressLogger : IProgressReporter { private ProgressOperation currentOperation; - private int progressPercentage; + private int _percentage; + private int _current; + private int _total; /// - public void Report(ProgressOperation operation, SystemFile file, int current, int total) + public void Report(ProgressOperation operation, SystemFile file) { - int num = (int)(current / (double)total * 100.0 + 0.5); + int num = (int)(_current++ / (double)_total * 100.0 + 0.5); if (currentOperation != operation) { - progressPercentage = -1; + _percentage = -1; currentOperation = operation; } - if (progressPercentage == num || num % 10 != 0) return; - Log.Information($"{operation}: {current} out of {total} ({progressPercentage}%)"); - progressPercentage = num; + if (_percentage == num || num % 10 != 0) return; + Log.Information($"{operation}: {_current} out of {_total} ({_percentage}%)"); + _percentage = num; + } + + /// + public void Reset() + { + _current = 0; } /// public void Failed(Exception exception, SystemFile file) { - Log.Error($"{exception.GetType().FullName}: {exception.Message}. Failed to upload file: '{file.LocalPath}'"); + Log.Error($"{exception.GetType().FullName}: {exception.Message} Failed to upload file: '{file.LocalPath}'"); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index d095a33..c3bdfb2 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -69,6 +69,15 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options }); } + /// + public async Task DownloadFileAsync(string sourcePath, string destinationPath) + { + await using FileStream openStream = File.OpenRead(destinationPath); + await using FileStream createStream = File.Create(sourcePath); + await using GZipStream gzipStream = new GZipStream(openStream, CompressionMode.Decompress); + await gzipStream.CopyToAsync(createStream); + } + /// public Task ExistsAsync(string path) { @@ -98,6 +107,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string? parent = Path.GetDirectoryName(file.RemotePath); if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); + progress.Report(ProgressOperation.Uploading, file); await using FileStream openStream = File.OpenRead(file.LocalPath); await using FileStream createStream = File.Create(file.RemotePath); await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); @@ -107,9 +117,31 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } catch (Exception ex) { - progress.Failed(ex, file); + Log.Error(ex.GetBaseException().ToString()); } }); } + + /// + public async Task UploadFileAsync(string sourcePath, string destinationPath) + { + try + { + if (await ExistsAsync(destinationPath)) File.SetAttributes(destinationPath, ~FileAttributes.ReadOnly & File.GetAttributes(destinationPath)); + string? parent = Path.GetDirectoryName(destinationPath); + if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); + + await using FileStream openStream = File.OpenRead(sourcePath); + await using FileStream createStream = File.Create(destinationPath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await openStream.CopyToAsync(gzipStream); + + File.SetAttributes(destinationPath, File.GetAttributes(destinationPath) | FileAttributes.ReadOnly); + } + catch (Exception ex) + { + Log.Error(ex.GetBaseException().ToString()); + } + } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/IFileSystem.cs b/Parallel.Core/IO/FileSystem/IFileSystem.cs index 7346d81..2bd797a 100644 --- a/Parallel.Core/IO/FileSystem/IFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/IFileSystem.cs @@ -40,6 +40,13 @@ public interface IFileSystem : IDisposable /// Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress); + /// + /// Downloads a file from the associated file system. + /// + /// + /// + Task DownloadFileAsync(string sourcePath, string destinationPath); + /// /// Checks if a path exists on the associated file system. /// @@ -60,5 +67,12 @@ public interface IFileSystem : IDisposable /// /// Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress); + + /// + /// Uploads a file to the associated file system. + /// + /// + /// + Task UploadFileAsync(string sourcePath, string destinationPath); } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs index f00c498..964bca6 100644 --- a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs @@ -75,11 +75,19 @@ public async Task DeleteFileAsync(string path) } } + /// public Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress) { throw new NotImplementedException(); } + /// + public Task DownloadFileAsync(string sourcePath, string destinationPath) + { + throw new NotImplementedException(); + } + + /// public async Task ExistsAsync(string path) { return _client.IsConnected && await _client.ExistsAsync(path); @@ -109,7 +117,7 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres { SystemFile file = files[i]; Stopwatch sw = new Stopwatch(); - progress.Report(ProgressOperation.Uploading, file, i, files.Length); + progress.Report(ProgressOperation.Uploading, file); if (await _client.ExistsAsync(file.RemotePath)) _client.ChangePermissions(file.RemotePath, 644); string parentDir = string.Empty; @@ -131,5 +139,27 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres Log.Debug($"Uploaded '{file.RemotePath}' in {sw.ElapsedMilliseconds}ms"); } } + + /// + public async Task UploadFileAsync(string sourcePath, string destinationPath) + { + if (await ExistsAsync(destinationPath)) _client.ChangePermissions(destinationPath, 644); + + string parentDir = string.Empty; + foreach (string subPath in destinationPath.Split('/')) + { + parentDir += $"/{subPath}"; + if (!await _client.ExistsAsync(parentDir)) + { + await _client.CreateDirectoryAsync(parentDir); + } + } + + await using SftpFileStream createStream = _client.Create(destinationPath); + await using FileStream openStream = File.OpenRead(sourcePath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await openStream.CopyToAsync(gzipStream); + _client.ChangePermissions(destinationPath, 444); + } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index a1f388e..6d4c7a2 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -29,17 +29,19 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter SystemFile[] backupFiles = files.Where(f => !f.Deleted).ToArray(); Log.Information($"Backing up {backupFiles.Length} files..."); await FileSystem.UploadFilesAsync(backupFiles, progress); + + progress.Reset(); await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { if (file.Deleted) { - progress.Report(ProgressOperation.Archiving, file, 0, files.Length); + progress.Report(ProgressOperation.Archiving, file); await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); await Database.AddFileAsync(file); } else { - progress.Report(ProgressOperation.Syncing, file, 0, files.Length); + progress.Report(ProgressOperation.Syncing, file); SystemFile remote = await FileSystem.GetFileAsync(file.RemotePath); if (remote is not null) { From c0a5f2b0f9742142ad030704140d0d9dc5b51545 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Sun, 26 Oct 2025 04:28:53 -0500 Subject: [PATCH 6/8] Now can push via SSH --- Parallel.Cli/Program.cs | 2 +- Parallel.Cli/Utils/ProgressReport.cs | 2 +- .../IO/FileSystem/DotNetFileSystem.cs | 20 ++++-- Parallel.Core/IO/FileSystem/IFileSystem.cs | 2 +- Parallel.Core/IO/FileSystem/SftpFileSystem.cs | 66 +++++++++---------- Parallel.Core/IO/Syncing/FileSyncManager.cs | 4 +- Parallel.Core/Models/SystemFile.cs | 10 +++ Parallel.Core/Settings/LocalVaultConfig.cs | 5 +- 8 files changed, 64 insertions(+), 47 deletions(-) diff --git a/Parallel.Cli/Program.cs b/Parallel.Cli/Program.cs index b5b3c43..83caf15 100644 --- a/Parallel.Cli/Program.cs +++ b/Parallel.Cli/Program.cs @@ -18,7 +18,7 @@ public static async Task Main(string[] args) string logFile = Path.Combine(PathBuilder.ProgramData, "Logs", "latest.txt"); if (File.Exists(logFile)) File.Delete(logFile); - Log.Logger = new LoggerConfiguration().MinimumLevel.Debug().WriteTo.File(logFile).CreateLogger(); + Log.Logger = new LoggerConfiguration().MinimumLevel.Warning().WriteTo.File(logFile).CreateLogger(); AssemblyName assembly = Assembly.GetExecutingAssembly().GetName(); Log.Information($"{assembly.Name} [Version {assembly.Version}]"); diff --git a/Parallel.Cli/Utils/ProgressReport.cs b/Parallel.Cli/Utils/ProgressReport.cs index e9c587c..5fe0409 100644 --- a/Parallel.Cli/Utils/ProgressReport.cs +++ b/Parallel.Cli/Utils/ProgressReport.cs @@ -22,7 +22,7 @@ public ProgressReport(LocalVaultConfig localVault, int totalFiles) public void Report(ProgressOperation operation, SystemFile file) { int percent = _current++ * 100 / _total; - CommandLine.WriteLine($"[{percent}%] <{_localVault.Id}> {operation}: {file.LocalPath}"); + CommandLine.WriteLine($"[{_localVault.Id}] ({percent}%) {operation}: {file.LocalPath}"); } /// diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index c3bdfb2..b41630c 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -85,21 +85,29 @@ public Task ExistsAsync(string path) } /// - public Task GetFileAsync(string path) + public Task GetFileAsync(string path) { + if(!File.Exists(path)) return Task.FromResult(null); + FileInfo fi = new(path); - return Task.FromResult(new SystemFile(path) + SystemFile file = new SystemFile(path) { Name = fi.Name, RemotePath = fi.FullName, RemoteSize = fi.Length - }); + }; + return Task.FromResult(file); } /// public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress) { - await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + // await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + // { + // + // }); + + foreach (SystemFile file in files) { try { @@ -111,7 +119,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options await using FileStream openStream = File.OpenRead(file.LocalPath); await using FileStream createStream = File.Create(file.RemotePath); await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream, ct); + await openStream.CopyToAsync(gzipStream); File.SetAttributes(file.RemotePath, File.GetAttributes(file.RemotePath) | FileAttributes.ReadOnly); } @@ -119,7 +127,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options { Log.Error(ex.GetBaseException().ToString()); } - }); + } } /// diff --git a/Parallel.Core/IO/FileSystem/IFileSystem.cs b/Parallel.Core/IO/FileSystem/IFileSystem.cs index 2bd797a..76ebe7f 100644 --- a/Parallel.Core/IO/FileSystem/IFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/IFileSystem.cs @@ -59,7 +59,7 @@ public interface IFileSystem : IDisposable /// /// /// - Task GetFileAsync(string path); + Task GetFileAsync(string path); /// /// Uploads an array of files to the associated file system. diff --git a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs index 964bca6..64ae552 100644 --- a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs @@ -76,9 +76,15 @@ public async Task DeleteFileAsync(string path) } /// - public Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress) + public async Task DownloadFilesAsync(SystemFile[] files, IProgressReporter progress) { - throw new NotImplementedException(); + await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + { + await using SftpFileStream openStream = _client.OpenRead(file.RemotePath); + await using FileStream createStream = File.Create(file.LocalPath); + await using GZipStream gzipStream = new GZipStream(openStream, CompressionMode.Decompress); + await gzipStream.CopyToAsync(createStream, ct); + }); } /// @@ -94,49 +100,41 @@ public async Task ExistsAsync(string path) } /// - public async Task GetFileAsync(string path) + public async Task GetFileAsync(string path) { - SystemFile file = new SystemFile(path); - if (await ExistsAsync(path)) - { - ISftpFile sf = _client.Get(path); - file = new SystemFile(sf.FullName) - { - Name = sf.Name, - RemoteSize = sf.Length, - }; - } + if (!await ExistsAsync(path)) return null; - return file; + ISftpFile sf = _client.Get(path); + return new SystemFile(sf.Name, sf.FullName, sf.Length, sf.LastWriteTime); } /// public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress) { - for (int i = 0; i < files.Length; i++) + foreach (SystemFile file in files) { - SystemFile file = files[i]; - Stopwatch sw = new Stopwatch(); - progress.Report(ProgressOperation.Uploading, file); - if (await _client.ExistsAsync(file.RemotePath)) _client.ChangePermissions(file.RemotePath, 644); - - string parentDir = string.Empty; - foreach (string subPath in file.RemotePath.Split('/')) + try { - parentDir += $"/{subPath}"; - if (!await _client.ExistsAsync(parentDir)) - { - await _client.CreateDirectoryAsync(parentDir); - } - } + Stopwatch sw = new Stopwatch(); + progress.Report(ProgressOperation.Uploading, file); + if (await _client.ExistsAsync(file.RemotePath)) _client.ChangePermissions(file.RemotePath, 644); - await using SftpFileStream createStream = _client.Create(file.RemotePath); - await using FileStream openStream = File.OpenRead(file.LocalPath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream); - _client.ChangePermissions(file.RemotePath, 444); + string[] subDirs = file.RemotePath.Split('/'); + string parentDir = string.Join("/", subDirs.Take(subDirs.Length - 1)); + if(!await _client.ExistsAsync(parentDir)) await CreateDirectoryAsync(parentDir); - Log.Debug($"Uploaded '{file.RemotePath}' in {sw.ElapsedMilliseconds}ms"); + await using SftpFileStream createStream = _client.Create(file.RemotePath); + await using FileStream openStream = File.OpenRead(file.LocalPath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await openStream.CopyToAsync(gzipStream); + _client.ChangePermissions(file.RemotePath, 444); + + Log.Debug($"Uploaded '{file.RemotePath}' in {sw.ElapsedMilliseconds}ms"); + } + catch (Exception ex) + { + Log.Error(ex.GetBaseException().ToString()); + } } } diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 6d4c7a2..c2a135a 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -41,8 +41,8 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } else { - progress.Report(ProgressOperation.Syncing, file); - SystemFile remote = await FileSystem.GetFileAsync(file.RemotePath); + //progress.Report(ProgressOperation.Syncing, file); + SystemFile? remote = await FileSystem.GetFileAsync(file.RemotePath); if (remote is not null) { file.RemoteSize = remote.RemoteSize; diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index 2b9f6a9..9ad23ca 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -140,6 +140,16 @@ public SystemFile(string id, string name, string localpath, string remotepath, l CheckSum = checksum; } + public SystemFile(string name, string remotePath, long length, DateTime lastWriteTime) + { + Name = name; + RemotePath = remotePath; + RemoteSize = length; + LastWrite = new UnixTime(lastWriteTime); + } + + //public SystemFile() { } + /// /// Determines if this instance and another have the same values. /// diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs index 9d9ef39..76f5652 100644 --- a/Parallel.Core/Settings/LocalVaultConfig.cs +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -1,5 +1,6 @@ // Copyright 2025 Kyle Ebbinga +using Parallel.Core.IO.FileSystem; using Parallel.Core.Security; namespace Parallel.Core.Settings @@ -12,12 +13,12 @@ public class LocalVaultConfig /// /// A unique hash used to identify the vault. /// - public string Id { get; } = HashGenerator.GenerateHash(12, true); + public string Id { get; } /// /// The name of the vault. /// - public string Name { get; set; } = "Default"; + public string Name { get; set; } /// /// The credentials needed to log in to the associated . From ab92e55b5b8e37e86b3948a1cb10ee8960c032d1 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Mon, 10 Nov 2025 04:24:07 -0600 Subject: [PATCH 7/8] Various performance increases --- Parallel.Cli/Commands/PushCommand.cs | 2 +- Parallel.Cli/Commands/UnzipCommand.cs | 4 ++-- .../IO/FileSystem/DotNetFileSystem.cs | 11 +++-------- Parallel.Core/IO/Scanning/FileScanner.cs | 18 ++++++++++++++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 1b6204e..4662efc 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -41,7 +41,7 @@ public PushCommand() : base("push", "Pushes changed files to vaults.") }, _sourceArg, _configOpt, _verboseOpt); } - private async Task SyncSystemAsync() + private Task SyncSystemAsync() { throw new NotImplementedException(); } diff --git a/Parallel.Cli/Commands/UnzipCommand.cs b/Parallel.Cli/Commands/UnzipCommand.cs index 97c0030..396a646 100644 --- a/Parallel.Cli/Commands/UnzipCommand.cs +++ b/Parallel.Cli/Commands/UnzipCommand.cs @@ -12,7 +12,7 @@ public class UnzipCommand : Command private readonly Argument sourceArg = new("path", "The source path of files to unzip."); private readonly Option keepOpt = new(["--keep", "-k"], "If the original files should be kept."); - private Stopwatch _sw; + private Stopwatch? _sw; private readonly List _tasks = new List(); private int _totalTasks = 0; @@ -58,7 +58,7 @@ private void DecompressFile(string path, bool keep) } } - CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw.Elapsed, ConsoleColor.DarkGray); + CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw?.Elapsed ?? TimeSpan.Zero, ConsoleColor.DarkGray); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index b41630c..799f181 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -102,12 +102,7 @@ public Task ExistsAsync(string path) /// public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress) { - // await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => - // { - // - // }); - - foreach (SystemFile file in files) + await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { try { @@ -119,7 +114,7 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres await using FileStream openStream = File.OpenRead(file.LocalPath); await using FileStream createStream = File.Create(file.RemotePath); await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream); + await openStream.CopyToAsync(gzipStream, ct); File.SetAttributes(file.RemotePath, File.GetAttributes(file.RemotePath) | FileAttributes.ReadOnly); } @@ -127,7 +122,7 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres { Log.Error(ex.GetBaseException().ToString()); } - } + }); } /// diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index 993ce9f..2ad7211 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -58,7 +58,7 @@ public async Task GetFileChangesAsync(string path, string[] ignore List scannedFiles = new List(); HashSet localFiles = FileScanner.GetFiles(path, ".", ignoreFolders).ToHashSet(); IEnumerable remoteFiles = await _db.GetFilesAsync(path, false); - foreach (SystemFile remoteFile in remoteFiles) + await System.Threading.Tasks.Parallel.ForEachAsync(remoteFiles, ParallelConfig.Options, async (remoteFile, ct) => { if (File.Exists(remoteFile.LocalPath) && remoteFile.RemotePath != null) { @@ -85,17 +85,27 @@ public async Task GetFileChangesAsync(string path, string[] ignore remoteFile.Deleted = true; scannedFiles.Add(remoteFile); } - } + }); + + // foreach (SystemFile remoteFile in remoteFiles) + // { + // + // } Log.Debug($"{localFiles.Count} files are untracked! Adding..."); - foreach (var file in localFiles) + // foreach (var file in localFiles) + // { + // + // } + + await System.Threading.Tasks.Parallel.ForEachAsync(localFiles, ParallelConfig.Options, async (file, ct) => { if (File.Exists(file) && !IsIgnored(file, ignoreFolders)) { Log.Debug($"Created -> {file}"); scannedFiles.Add(new SystemFile(file) { RemotePath = PathBuilder.Remote(file, _config) }); } - } + }); Log.Debug($"{localFiles.Count} files remaining."); Log.Information($"Found {scannedFiles.Count:N0} changes in '{path}'"); From 3057deedcaaede5939cf190ef734e44a25911b7e Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Mon, 10 Nov 2025 04:25:05 -0600 Subject: [PATCH 8/8] Added new configuration for analyzing builds --- Parallel.Cli/Parallel.Cli.csproj | 35 +++++++++++++++++++++++------- Parallel.Core/Parallel.Core.csproj | 2 ++ Parallel.sln | 5 +++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/Parallel.Cli/Parallel.Cli.csproj b/Parallel.Cli/Parallel.Cli.csproj index d042abc..edd0157 100644 --- a/Parallel.Cli/Parallel.Cli.csproj +++ b/Parallel.Cli/Parallel.Cli.csproj @@ -3,6 +3,7 @@ Exe net9.0 + Debug;Release;Analyze parallel-red.ico enable enable @@ -15,24 +16,42 @@ $(Company) + + $(DefineConstants);TRACE + true + true + + + $(DefineConstants);DEBUG;TRACE + false + true + full + + + False + True + True + True + + - - - - + + + + - - + + - + - + diff --git a/Parallel.Core/Parallel.Core.csproj b/Parallel.Core/Parallel.Core.csproj index aa8cd3a..950da5d 100644 --- a/Parallel.Core/Parallel.Core.csproj +++ b/Parallel.Core/Parallel.Core.csproj @@ -13,6 +13,8 @@ Parallel.Core True Parallel.Core + Debug;Release;Analyze + AnyCPU diff --git a/Parallel.sln b/Parallel.sln index 329e4be..9ffbb75 100644 --- a/Parallel.sln +++ b/Parallel.sln @@ -11,16 +11,21 @@ Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU + Analyze|Any CPU = Analyze|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {1391ED00-10D5-417A-982B-0A9A6A2295D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1391ED00-10D5-417A-982B-0A9A6A2295D4}.Debug|Any CPU.Build.0 = Debug|Any CPU {1391ED00-10D5-417A-982B-0A9A6A2295D4}.Release|Any CPU.ActiveCfg = Release|Any CPU {1391ED00-10D5-417A-982B-0A9A6A2295D4}.Release|Any CPU.Build.0 = Release|Any CPU + {1391ED00-10D5-417A-982B-0A9A6A2295D4}.Analyze|Any CPU.ActiveCfg = Analyze|Any CPU + {1391ED00-10D5-417A-982B-0A9A6A2295D4}.Analyze|Any CPU.Build.0 = Analyze|Any CPU {4BFE65E9-9534-4C85-B59F-1F64A998C76D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4BFE65E9-9534-4C85-B59F-1F64A998C76D}.Debug|Any CPU.Build.0 = Debug|Any CPU {4BFE65E9-9534-4C85-B59F-1F64A998C76D}.Release|Any CPU.ActiveCfg = Release|Any CPU {4BFE65E9-9534-4C85-B59F-1F64A998C76D}.Release|Any CPU.Build.0 = Release|Any CPU + {4BFE65E9-9534-4C85-B59F-1F64A998C76D}.Analyze|Any CPU.ActiveCfg = Analyze|Any CPU + {4BFE65E9-9534-4C85-B59F-1F64A998C76D}.Analyze|Any CPU.Build.0 = Analyze|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE