From c41d0ba9766479d3a0de48fb6c83cb34828cb8a6 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 04:00:22 -0600 Subject: [PATCH 01/17] New remap command for remapping system paths --- Parallel.Cli/Commands/RemapCommand.cs | 74 +++++++++++++++++++ Parallel.Cli/Commands/SyncCommand.cs | 8 ++ Parallel.Cli/Commands/VaultsCommand.cs | 7 ++ Parallel.Cli/Commands/ZipCommand.cs | 4 +- .../Database/Contexts/SqliteContext.cs | 16 ++++ Parallel.Core/Database/IDatabase.cs | 3 + Parallel.Core/Models/SystemFile.cs | 2 +- 7 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 Parallel.Cli/Commands/SyncCommand.cs diff --git a/Parallel.Cli/Commands/RemapCommand.cs b/Parallel.Cli/Commands/RemapCommand.cs index 459193d..930af3a 100644 --- a/Parallel.Cli/Commands/RemapCommand.cs +++ b/Parallel.Cli/Commands/RemapCommand.cs @@ -1,13 +1,87 @@ // Copyright 2025 Kyle Ebbinga using System.CommandLine; +using System.ComponentModel.DataAnnotations.Schema; +using System.Diagnostics; +using Parallel.Cli.Utils; +using Parallel.Core.IO.Syncing; +using Parallel.Core.Models; +using Parallel.Core.Security; +using Parallel.Core.Settings; +using SQLitePCL; namespace Parallel.Cli.Commands { public class RemapCommand : Command { + private readonly Argument _sourceArg = new("source", "The source path to change."); + private readonly Argument _targetArg = new("target", "The target path to change to."); + private readonly Option _optionOpt = new("config", "The vault configuration to use."); + + private Stopwatch _sw = new Stopwatch(); + public RemapCommand() : base("remap", "Remaps paths in the vault.") { + this.AddArgument(_sourceArg); + this.AddArgument(_targetArg); + this.SetHandler(async (config, source, target) => + { + _sw = Stopwatch.StartNew(); + LocalVaultConfig? vault = ParallelConfig.Load().Vaults.FirstOrDefault(); + if (!string.IsNullOrEmpty(config)) vault = ParallelConfig.GetVault(config); + if (vault == null) + { + CommandLine.WriteLine($"Unable to find vault with name: '{vault}'", ConsoleColor.Yellow); + return; + } + + if (string.IsNullOrEmpty(source) || string.IsNullOrEmpty(target)) + { + CommandLine.WriteLine("The source and target paths must be specified!", ConsoleColor.Yellow); + return; + } + + await RemapPathAsync(vault, source, target); + }, _optionOpt, _sourceArg, _targetArg); + + } + + private async Task RemapPathAsync(LocalVaultConfig vault, string source, string target) + { + ISyncManager syncManager = SyncManager.CreateNew(vault); + if (!await syncManager.ConnectAsync()) + { + CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); + return; + } + + CommandLine.WriteLine($"Locating files for remapping...", ConsoleColor.DarkGray); + IEnumerable files = await syncManager.Database.GetFilesAsync(source); + if (!files.Any()) + { + CommandLine.WriteLine(vault, "No files were found!", ConsoleColor.Yellow); + return; + } + + int progress = 0; + int total = files.Count(); + await System.Threading.Tasks.Parallel.ForEachAsync(files, async (file, ct) => + { + CommandLine.ProgressBar(progress++, total, _sw.Elapsed, ConsoleColor.DarkGray); + + string newPath = file.LocalPath.Replace(source, target); + string newId = HashGenerator.CreateSHA1(newPath); + await syncManager.Database.RemapObjectsAsync(file.Id, newId); + + //CommandLine.WriteLine(vault, $"Remapping '{file.LocalPath}' to '{newPath}'"); + await syncManager.Database.RemoveFileAsync(file); + + file.Id = newId; + file.LocalPath = newPath; + await syncManager.Database.AddFileAsync(file); + }); + + CommandLine.WriteLine(vault, $"Successfully remapped {files.Count():N0} files to '{target}'.", ConsoleColor.Green); } } } \ No newline at end of file diff --git a/Parallel.Cli/Commands/SyncCommand.cs b/Parallel.Cli/Commands/SyncCommand.cs new file mode 100644 index 0000000..0d1cd5b --- /dev/null +++ b/Parallel.Cli/Commands/SyncCommand.cs @@ -0,0 +1,8 @@ +// Copyright 2025 Kyle Ebbinga + +namespace Parallel.Cli.Commands +{ + public class SyncCommand + { + } +} \ No newline at end of file diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index 8ad97f2..de0d753 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -18,6 +18,7 @@ public class VaultsCommand : Command private Command addCmd = new("add", "Adds a new vault configuration."); private Command editCmd = new("edit", "Edits a vault configuration."); + private readonly Command findCmd = new("find", "Finds vault configurations in a location."); private Command viewCmd = new("view", "Shows the vault configuration."); private Command setCmd = new("set", "Sets a new vault configuration."); private Command delCmd = new("delete", "Deletes a vault configuration."); @@ -69,6 +70,12 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") CommandLine.WriteLine($"Saved new storage vault: '{localVault.Name}' ({localVault.Id})"); }); + this.AddCommand(findCmd); + findCmd.SetHandler(() => + { + + }); + this.AddCommand(viewCmd); viewCmd.AddArgument(configArg); viewCmd.SetHandler(async (vault) => diff --git a/Parallel.Cli/Commands/ZipCommand.cs b/Parallel.Cli/Commands/ZipCommand.cs index 83722f7..ef6b354 100644 --- a/Parallel.Cli/Commands/ZipCommand.cs +++ b/Parallel.Cli/Commands/ZipCommand.cs @@ -33,12 +33,12 @@ public ZipCommand() : base("zip", "Zips files in a directory.") return; } - CommandLine.WriteLine($"Zipping {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); + CommandLine.WriteLine($"Zipping {files.Length:N0} files...", ConsoleColor.DarkGray); _totalTasks = files.Length; _tasks.AddRange(files.Select(file => Task.Run(() => CompressFile(file, keep)))); await Task.WhenAll(_tasks); - CommandLine.WriteLine($"Successfully zipped {files.Length.ToString("N0")} files in {_sw.Elapsed}.", ConsoleColor.Green); + CommandLine.WriteLine($"Successfully zipped {files.Length:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); }, sourceArg, keepOpt); } diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 1ab16e8..10c9c67 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -59,6 +59,14 @@ public async Task AddFileAsync(SystemFile file) return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.CheckSum }) > 0; } + /// + public async Task RemoveFileAsync(SystemFile file) + { + using IDbConnection connection = CreateConnection(); + string sql = $"DELETE FROM files WHERE id = @Id;"; + await connection.ExecuteAsync(sql, new { file.Id }); + } + public async Task GetLocalSizeAsync() { using IDbConnection connection = CreateConnection(); @@ -148,6 +156,14 @@ public async Task> GetObjectsAsync(string id) return await connection.QueryAsync(sql, new { id }); } + /// + public async Task RemapObjectsAsync(string oldId, string newId) + { + using IDbConnection connection = CreateConnection(); + string sql = "UPDATE objects SET id = @newId WHERE id = @oldId;"; + return await connection.ExecuteAsync(sql, new { oldId, newId }) > 0; + } + #endregion } } \ No newline at end of file diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index fddf31b..2bd0fba 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -75,6 +75,8 @@ public interface IDatabase /// True if successful, false otherwise Task AddFileAsync(SystemFile file); + Task RemoveFileAsync(SystemFile file); + Task> GetFilesAsync(string path); Task> GetFilesAsync(string path, bool deleted); Task GetFileAsync(string path); @@ -108,5 +110,6 @@ public interface IDatabase #endregion + Task RemapObjectsAsync(string oldId, string newId); } } \ No newline at end of file diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index 6410b0e..0749743 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -16,7 +16,7 @@ public class SystemFile /// /// The unique identifier of the file. /// - public string Id { get; } = string.Empty; + public string Id { get; set; } = string.Empty; /// /// The name of the file. From edaed6fcaca134edf270ce3302d922cba73ae3cf Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 04:59:26 -0600 Subject: [PATCH 02/17] Various clean ups --- Parallel.Cli/Commands/PullCommand.cs | 3 ++- Parallel.Cli/Commands/RemapCommand.cs | 9 +++++---- Parallel.Cli/Commands/UnzipCommand.cs | 2 +- Parallel.Cli/Commands/ZipCommand.cs | 2 +- Parallel.Cli/Utils/CommandLine.cs | 8 +++----- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 2 -- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Parallel.Cli/Commands/PullCommand.cs b/Parallel.Cli/Commands/PullCommand.cs index 0b2c895..6f6c1c9 100644 --- a/Parallel.Cli/Commands/PullCommand.cs +++ b/Parallel.Cli/Commands/PullCommand.cs @@ -54,6 +54,7 @@ private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force return; } + CommandLine.WriteLine(vault, $"Scanning for files in {path}...", ConsoleColor.DarkGray); IEnumerable files = await syncManager.Database.GetFilesAsync(fullPath); if (!files.Any()) { @@ -69,7 +70,7 @@ private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force }); Log.Debug($"Pulling {pullFiles.Count} files..."); - await syncManager.PullFilesAsync(pullFiles.ToArray(), new ProgressLogger()); + await syncManager.PullFilesAsync(pullFiles.ToArray(), new ProgressReport(vault, files.Count())); CommandLine.WriteLine(vault, $"Successfully pulled {pullFiles.Count:N0} files from '{vault.FileSystem.RootDirectory}'.", ConsoleColor.Green); } diff --git a/Parallel.Cli/Commands/RemapCommand.cs b/Parallel.Cli/Commands/RemapCommand.cs index 930af3a..8641ae5 100644 --- a/Parallel.Cli/Commands/RemapCommand.cs +++ b/Parallel.Cli/Commands/RemapCommand.cs @@ -55,7 +55,7 @@ private async Task RemapPathAsync(LocalVaultConfig vault, string source, string return; } - CommandLine.WriteLine($"Locating files for remapping...", ConsoleColor.DarkGray); + CommandLine.WriteLine(vault, $"Scanning for files in {source}...", ConsoleColor.DarkGray); IEnumerable files = await syncManager.Database.GetFilesAsync(source); if (!files.Any()) { @@ -65,15 +65,15 @@ private async Task RemapPathAsync(LocalVaultConfig vault, string source, string int progress = 0; int total = files.Count(); + + CommandLine.WriteLine(vault, $"Remapping '{source}' to '{target}'..."); await System.Threading.Tasks.Parallel.ForEachAsync(files, async (file, ct) => { - CommandLine.ProgressBar(progress++, total, _sw.Elapsed, ConsoleColor.DarkGray); + CommandLine.ProgressBar(progress++, total, _sw.Elapsed); string newPath = file.LocalPath.Replace(source, target); string newId = HashGenerator.CreateSHA1(newPath); await syncManager.Database.RemapObjectsAsync(file.Id, newId); - - //CommandLine.WriteLine(vault, $"Remapping '{file.LocalPath}' to '{newPath}'"); await syncManager.Database.RemoveFileAsync(file); file.Id = newId; @@ -81,6 +81,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, async (file, ct) => await syncManager.Database.AddFileAsync(file); }); + await syncManager.DisconnectAsync(); CommandLine.WriteLine(vault, $"Successfully remapped {files.Count():N0} files to '{target}'.", ConsoleColor.Green); } } diff --git a/Parallel.Cli/Commands/UnzipCommand.cs b/Parallel.Cli/Commands/UnzipCommand.cs index 396a646..db0aae5 100644 --- a/Parallel.Cli/Commands/UnzipCommand.cs +++ b/Parallel.Cli/Commands/UnzipCommand.cs @@ -58,7 +58,7 @@ private void DecompressFile(string path, bool keep) } } - CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw?.Elapsed ?? TimeSpan.Zero, ConsoleColor.DarkGray); + CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw?.Elapsed ?? TimeSpan.Zero); } } } \ No newline at end of file diff --git a/Parallel.Cli/Commands/ZipCommand.cs b/Parallel.Cli/Commands/ZipCommand.cs index ef6b354..6d065f9 100644 --- a/Parallel.Cli/Commands/ZipCommand.cs +++ b/Parallel.Cli/Commands/ZipCommand.cs @@ -60,7 +60,7 @@ private void CompressFile(string path, bool keep) } } - CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw.Elapsed, ConsoleColor.DarkGray); + CommandLine.ProgressBar(_tasks.Count(t => t.IsCompleted), _totalTasks, _sw.Elapsed); } } } \ No newline at end of file diff --git a/Parallel.Cli/Utils/CommandLine.cs b/Parallel.Cli/Utils/CommandLine.cs index 3f16a0c..0e01b11 100644 --- a/Parallel.Cli/Utils/CommandLine.cs +++ b/Parallel.Cli/Utils/CommandLine.cs @@ -127,17 +127,15 @@ public static void WriteLine(object value, ConsoleColor color = ConsoleColor.Gra } } - public static void ProgressBar(double part, double total, TimeSpan elapsed, ConsoleColor color = ConsoleColor.Gray) + public static void ProgressBar(double part, double total, TimeSpan elapsed) { double percent = part / total; string percentStr = $"> Progress: {Convert.ToInt32(percent * 100).ToString("D2")}%"; TimeSpan remaining; double remainingMs = elapsed.TotalMilliseconds * (total - part) / part; - if (remainingMs <= TimeSpan.MaxValue.TotalMilliseconds) - remaining = TimeSpan.FromMilliseconds(remainingMs); - else - remaining = TimeSpan.MaxValue; + if (remainingMs <= TimeSpan.MaxValue.TotalMilliseconds) remaining = TimeSpan.FromMilliseconds(remainingMs); + else remaining = TimeSpan.MaxValue; string remainingStr = $"{remaining.Hours:00}:{remaining.Minutes:00}:{remaining.Seconds:00} remaining"; int barWidth = Console.WindowWidth - percentStr.Length - remainingStr.Length - 4; diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index 3c6f417..ade52c5 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -83,10 +83,8 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options { string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); - if (await FileSystem.ExistsAsync(remotePath)) { - Log.Debug($"Downloading object: {hash}"); await FileSystem.DownloadStreamAsync(fs, remotePath); } } From 6e89b4da851114e85d7d63838ee55f533ba7e42d Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 06:03:22 -0600 Subject: [PATCH 03/17] Cleaned up command line messages and reworked disk command to show new object storage savings --- Parallel.Cli/Commands/DiskCommand.cs | 13 ++- .../Database/Contexts/SqliteContext.cs | 8 ++ Parallel.Core/Database/IDatabase.cs | 3 +- .../Diagnostics/IProgressReporter.cs | 4 +- .../IO/FileSystem/DotNetFileSystem.cs | 1 - Parallel.Core/IO/FileSystem/SftpFileSystem.cs | 1 - Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 101 +++++++++++------- 7 files changed, 82 insertions(+), 49 deletions(-) diff --git a/Parallel.Cli/Commands/DiskCommand.cs b/Parallel.Cli/Commands/DiskCommand.cs index 4753ae6..2ed79a1 100644 --- a/Parallel.Cli/Commands/DiskCommand.cs +++ b/Parallel.Cli/Commands/DiskCommand.cs @@ -45,9 +45,15 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) IDatabase db = syncManager.Database; long localSize = await db.GetLocalSizeAsync(); - long remoteSize = await db.GetRemoteSizeAsync(); long totalLocalFiles = await db.GetTotalFilesAsync(false); long totalDeletedFiles = await db.GetTotalFilesAsync(true); + long totalObjects = await db.GetTotalObjectsAsync(); + + int chunkSize = ((ObjectSyncManager)syncManager).ChunkSize; + double expectedChunks = (double)localSize / chunkSize; + long expectedBytes = (long)(expectedChunks * chunkSize); + long actualBytes = totalObjects * chunkSize; + long savedBytes = expectedBytes - actualBytes; CommandLine.WriteLine($"Using vault '{vault.Name}' ({vault.Id}):"); CommandLine.WriteLine($"Service Type: {vault.FileSystem.Service}"); @@ -56,15 +62,14 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) CommandLine.WriteLine($"Local Files: {totalLocalFiles:N0}"); CommandLine.WriteLine($"Deleted Files: {totalDeletedFiles:N0}"); CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(localSize)}"); - CommandLine.WriteLine($"Remote Size: {Formatter.FromBytes(remoteSize)}"); - CommandLine.WriteLine($"Space Saved: {Math.Round((localSize - remoteSize) / (double)localSize * 100, 2)}%"); + CommandLine.WriteLine($"Total Objects: {totalObjects:N0}"); + CommandLine.WriteLine($"Space Saved: {Formatter.FromBytes(savedBytes)} ({Math.Round(savedBytes / (double)expectedBytes * 100, 1)}%)"); if (vault.FileSystem.Service.Equals(FileService.Local)) { DriveInfo drive = new(vault.FileSystem.RootDirectory); long diskUsage = drive.TotalSize - drive.TotalFreeSpace; CommandLine.WriteLine($"Total Usage: {Formatter.FromBytes(diskUsage)} ({Math.Round(diskUsage / (double)drive.TotalSize * 100, 1)}%)"); - CommandLine.WriteLine($"Disk Usage: {Formatter.FromBytes(diskUsage - remoteSize)} ({Math.Round((diskUsage - remoteSize) / (double)drive.TotalSize * 100, 1)}%)"); CommandLine.WriteLine($"Disk Free: {Formatter.FromBytes(drive.TotalFreeSpace)} ({Math.Round(drive.TotalFreeSpace / (double)drive.TotalSize * 100, 1)}%)"); CommandLine.WriteLine($"Disk Total: {Formatter.FromBytes(drive.TotalSize)}"); } diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 10c9c67..f3292cc 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -156,6 +156,14 @@ public async Task> GetObjectsAsync(string id) return await connection.QueryAsync(sql, new { id }); } + /// + public async Task GetTotalObjectsAsync() + { + using IDbConnection connection = CreateConnection(); + string sql = $"SELECT COUNT(*) FROM objects;"; + return await connection.QuerySingleOrDefaultAsync(sql); + } + /// public async Task RemapObjectsAsync(string oldId, string newId) { diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 2bd0fba..643c8ef 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -82,7 +82,6 @@ public interface IDatabase Task GetFileAsync(string path); Task GetLocalSizeAsync(); - Task GetRemoteSizeAsync(); Task GetTotalFilesAsync(bool deleted); #endregion @@ -108,6 +107,8 @@ public interface IDatabase Task AddObjectAsync(string id, string hash, int index); Task> GetObjectsAsync(string id); + Task GetTotalObjectsAsync(); + #endregion Task RemapObjectsAsync(string oldId, string newId); diff --git a/Parallel.Core/Diagnostics/IProgressReporter.cs b/Parallel.Core/Diagnostics/IProgressReporter.cs index 943e0da..be7934f 100644 --- a/Parallel.Core/Diagnostics/IProgressReporter.cs +++ b/Parallel.Core/Diagnostics/IProgressReporter.cs @@ -7,8 +7,8 @@ namespace Parallel.Core.Diagnostics public enum ProgressOperation { Archiving, - Downloading, - Uploading, + Pulling, + Pushing, Compressing, Decompressing, Syncing diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index 05d6f6d..31035fc 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -110,7 +110,6 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string? parent = Path.GetDirectoryName(file.RemotePath); if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); - progress.Report(ProgressOperation.Uploading, file); await using FileStream openStream = File.OpenRead(file.LocalPath); await using FileStream createStream = File.Create(file.RemotePath); await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); diff --git a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs index da67727..9841c27 100644 --- a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs @@ -119,7 +119,6 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres try { Stopwatch sw = new Stopwatch(); - progress.Report(ProgressOperation.Uploading, file); if (await _client.ExistsAsync(file.RemotePath)) _client.ChangePermissions(file.RemotePath, 644); string[] subDirs = file.RemotePath.Split('/'); diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index ade52c5..7b3bfdb 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -1,5 +1,6 @@ // Copyright 2025 Kyle Ebbinga +using System.Collections.Concurrent; using System.Reflection.Metadata; using Newtonsoft.Json.Linq; using Parallel.Core.Database; @@ -16,11 +17,12 @@ namespace Parallel.Core.IO.Syncing /// public class ObjectSyncManager : BaseSyncManager { + private static readonly ConcurrentDictionary _locks = new(); + /// /// The size, in bytes, to use for chunks of a file. /// - private static readonly int ChunkSize = 4194304; - + public readonly int ChunkSize = 4194304; /// /// Initializes a new instance of the class. @@ -33,39 +35,49 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter { await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - if (file.Deleted) - { - progress.Report(ProgressOperation.Archiving, file); - await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); - await Database.AddFileAsync(file); - } - else + try { - int bytesRead = 0; - byte[] buffer = new byte[ChunkSize]; - await using FileStream fs = File.OpenRead(file.LocalPath); - - int index = 0; - progress.Report(ProgressOperation.Uploading, file); - await Database.AddFileAsync(file); - - while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) + if (file.Deleted) { - await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); - string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); - await Database.AddObjectAsync(file.Id, hash, index); - index++; + progress.Report(ProgressOperation.Archiving, file); + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); + await Database.AddFileAsync(file); + } + else + { + int bytesRead = 0; + byte[] buffer = new byte[ChunkSize]; + await using FileStream fs = File.OpenRead(file.LocalPath); - string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); - string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); - string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - if (!await FileSystem.ExistsAsync(remotePath)) + int index = 0; + progress.Report(ProgressOperation.Pushing, file); + await Database.AddFileAsync(file); + + while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) { - if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); - await FileSystem.UploadStreamAsync(ms, remotePath); + await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); + string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); + await Database.AddObjectAsync(file.Id, hash, index); + index++; + + string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); + string remotePath = PathBuilder.Combine(parentDir, hash[4..]); + if (!await FileSystem.ExistsAsync(remotePath)) + { + if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); + await FileSystem.UploadStreamAsync(ms, remotePath); + } } + + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pushed); } } + catch (Exception ex) + { + Log.Error(ex.GetBaseException().ToString()); + progress.Failed(ex, file); + } }); } @@ -74,22 +86,31 @@ public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter { await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - progress.Report(ProgressOperation.Downloading, file); - string? parentDir = Path.GetDirectoryName(file.LocalPath); - if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); - - await using FileStream fs = File.Create(file.LocalPath); - foreach (string hash in await Database.GetObjectsAsync(file.Id)) + try { - string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); - string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); - if (await FileSystem.ExistsAsync(remotePath)) + progress.Report(ProgressOperation.Pulling, file); + string? parentDir = Path.GetDirectoryName(file.LocalPath); + if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); + + await using FileStream fs = File.Create(file.LocalPath); + foreach (string hash in await Database.GetObjectsAsync(file.Id)) { - await FileSystem.DownloadStreamAsync(fs, remotePath); + string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); + if (await FileSystem.ExistsAsync(remotePath)) + { + await FileSystem.DownloadStreamAsync(fs, remotePath); + } } - } - await fs.FlushAsync(ct); + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pulled); + await fs.FlushAsync(ct); + } + catch (Exception ex) + { + Log.Error(ex.GetBaseException().ToString()); + progress.Failed(ex, file); + } }); } } From ac4f1ed9b6421e5bb6b3e892a4c0ef7154f9ab34 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 07:15:13 -0600 Subject: [PATCH 04/17] To avoid confusion between the IFileSystem and the FileSystemCredentials, IFileSystem is now IStorageProvider --- Parallel.Cli/Commands/DiskCommand.cs | 9 +----- Parallel.Core/Database/IDatabase.cs | 4 +-- Parallel.Core/IO/Blobs/BlobStorage.cs | 2 +- Parallel.Core/IO/PathBuilder.cs | 2 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 20 ++++++------ Parallel.Core/IO/Syncing/FileSyncManager.cs | 6 ++-- Parallel.Core/IO/Syncing/ISyncManager.cs | 4 +-- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 10 +++--- .../Settings/FileSystemCredentials.cs | 2 +- Parallel.Core/Settings/LocalVaultConfig.cs | 2 +- .../IStorageProvider.cs} | 3 +- .../LocalStorageProvider.cs} | 17 +++++----- .../SshStorageProvider.cs} | 31 +++---------------- .../StorageProvider.cs} | 8 ++--- Parallel.Core/Utils/Formatter.cs | 9 ++++-- 15 files changed, 51 insertions(+), 78 deletions(-) rename Parallel.Core/{IO/FileSystem/IFileSystem.cs => Storage/IStorageProvider.cs} (95%) rename Parallel.Core/{IO/FileSystem/DotNetFileSystem.cs => Storage/LocalStorageProvider.cs} (88%) rename Parallel.Core/{IO/FileSystem/SftpFileSystem.cs => Storage/SshStorageProvider.cs} (80%) rename Parallel.Core/{IO/FileSystem/FileSystemManager.cs => Storage/StorageProvider.cs} (81%) diff --git a/Parallel.Cli/Commands/DiskCommand.cs b/Parallel.Cli/Commands/DiskCommand.cs index 2ed79a1..ba713cd 100644 --- a/Parallel.Cli/Commands/DiskCommand.cs +++ b/Parallel.Cli/Commands/DiskCommand.cs @@ -49,21 +49,14 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) long totalDeletedFiles = await db.GetTotalFilesAsync(true); long totalObjects = await db.GetTotalObjectsAsync(); - int chunkSize = ((ObjectSyncManager)syncManager).ChunkSize; - double expectedChunks = (double)localSize / chunkSize; - long expectedBytes = (long)(expectedChunks * chunkSize); - long actualBytes = totalObjects * chunkSize; - long savedBytes = expectedBytes - actualBytes; - CommandLine.WriteLine($"Using vault '{vault.Name}' ({vault.Id}):"); CommandLine.WriteLine($"Service Type: {vault.FileSystem.Service}"); CommandLine.WriteLine($"Root Directory: {vault.FileSystem.RootDirectory}"); CommandLine.WriteLine($"Managed Files: {(totalLocalFiles + totalDeletedFiles):N0}"); CommandLine.WriteLine($"Local Files: {totalLocalFiles:N0}"); CommandLine.WriteLine($"Deleted Files: {totalDeletedFiles:N0}"); - CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(localSize)}"); CommandLine.WriteLine($"Total Objects: {totalObjects:N0}"); - CommandLine.WriteLine($"Space Saved: {Formatter.FromBytes(savedBytes)} ({Math.Round(savedBytes / (double)expectedBytes * 100, 1)}%)"); + CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(localSize)}"); if (vault.FileSystem.Service.Equals(FileService.Local)) { diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 643c8ef..2ef3b69 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -109,8 +109,8 @@ public interface IDatabase Task GetTotalObjectsAsync(); - #endregion - Task RemapObjectsAsync(string oldId, string newId); + + #endregion } } \ No newline at end of file diff --git a/Parallel.Core/IO/Blobs/BlobStorage.cs b/Parallel.Core/IO/Blobs/BlobStorage.cs index 668a8a3..f34413f 100644 --- a/Parallel.Core/IO/Blobs/BlobStorage.cs +++ b/Parallel.Core/IO/Blobs/BlobStorage.cs @@ -23,7 +23,7 @@ public abstract class BlobStorage /// The temp directory to send chunked objects to. /// /// - public static async Task CreateManifestAsync(IFileSystem fileSystem, string sourcePath, string tempObjDir, IProgressReporter progress) + public static async Task CreateManifestAsync(IStorageProvider fileSystem, string sourcePath, string tempObjDir, IProgressReporter progress) { List chunkHashes = new List(); await using FileStream fs = File.OpenRead(sourcePath); diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index 55ad1c3..fff8c25 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -142,7 +142,7 @@ public static string GetDatabaseFile(LocalVaultConfig localVault) } /// - /// Builds the path on the remote . + /// Builds the path on the remote . /// /// /// diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index 4280bdf..f368ab2 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -27,7 +27,7 @@ public abstract class BaseSyncManager : ISyncManager public IDatabase Database { get; set; } /// - public IFileSystem FileSystem { get; set; } + public IStorageProvider Storage { get; set; } /// /// @@ -35,7 +35,7 @@ public abstract class BaseSyncManager : ISyncManager /// public BaseSyncManager(LocalVaultConfig localVault) { - FileSystem = FileSystemManager.CreateNew(localVault); + Storage = StorageProvider.CreateNew(localVault); LocalVault = localVault; } @@ -43,13 +43,13 @@ public BaseSyncManager(LocalVaultConfig localVault) public async Task ConnectAsync() { string root = PathBuilder.GetRootDirectory(LocalVault); - if (!await FileSystem.ExistsAsync(root)) + if (!await Storage.ExistsAsync(root)) { - await FileSystem.CreateDirectoryAsync(root); + await Storage.CreateDirectoryAsync(root); Log.Debug($"Created root directory: {root}"); } - if (!await FileSystem.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) + if (!await Storage.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) { RemoteVault = new RemoteVaultConfig(LocalVault); RemoteVault.IgnoreDirectories.Add(PathBuilder.GetRootDirectory(LocalVault)); @@ -59,7 +59,7 @@ public async Task ConnectAsync() } else { - await FileSystem.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new NullProgressReporter()); + await Storage.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new NullProgressReporter()); RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); if(config == null) return false; RemoteVault = config; @@ -67,7 +67,7 @@ public async Task ConnectAsync() Log.Debug($"Downloaded config file: {TempConfigFile}"); } - if (!await FileSystem.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) + if (!await Storage.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) { Database = new SqliteContext(TempDbFile); await Database.InitializeAsync(); @@ -76,7 +76,7 @@ public async Task ConnectAsync() } else { - await FileSystem.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new NullProgressReporter()); + await Storage.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new NullProgressReporter()); Database = new SqliteContext(TempDbFile); Log.Debug($"Downloaded db file: {TempDbFile}"); @@ -92,8 +92,8 @@ public async Task DisconnectAsync() Log.Debug($"Uploaded db file: {TempDbFile}"); SystemFile[] tempFiles = [new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault)), new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))]; - await FileSystem.UploadFilesAsync(tempFiles, new NullProgressReporter()); - FileSystem.Dispose(); + await Storage.UploadFilesAsync(tempFiles, new NullProgressReporter()); + Storage.Dispose(); } /// diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index d069ad6..7cfc03f 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -24,7 +24,7 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter if (!files.Any()) return; SystemFile[] backupFiles = files.Where(f => !f.Deleted).ToArray(); Log.Information($"Backing up {backupFiles.Length} files..."); - await FileSystem.UploadFilesAsync(backupFiles, progress); + await Storage.UploadFilesAsync(backupFiles, progress); progress.Reset(); await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => @@ -38,7 +38,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options else { progress.Report(ProgressOperation.Syncing, file); - SystemFile? remote = await FileSystem.GetFileAsync(file.RemotePath); + SystemFile? remote = await Storage.GetFileAsync(file.RemotePath); if (remote is not null) { file.RemoteSize = remote.RemoteSize; @@ -52,7 +52,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options /// public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter progress) { - await FileSystem.DownloadFilesAsync(files, progress); + await Storage.DownloadFilesAsync(files, progress); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index 15d1395..80f93f6 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -31,10 +31,10 @@ public interface ISyncManager /// /// The associated file system connection. /// - IFileSystem FileSystem { get; set; } + public IStorageProvider Storage { get; set; } /// - /// Establishes a connection to the associated and downloads the needed files. + /// Establishes a connection to the associated and downloads the needed files. /// Task ConnectAsync(); diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index 7b3bfdb..7d464cb 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -63,10 +63,10 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - if (!await FileSystem.ExistsAsync(remotePath)) + if (!await Storage.ExistsAsync(remotePath)) { - if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); - await FileSystem.UploadStreamAsync(ms, remotePath); + if (!await Storage.ExistsAsync(parentDir)) await Storage.CreateDirectoryAsync(parentDir); + await Storage.UploadStreamAsync(ms, remotePath); } } @@ -97,9 +97,9 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options { string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); - if (await FileSystem.ExistsAsync(remotePath)) + if (await Storage.ExistsAsync(remotePath)) { - await FileSystem.DownloadStreamAsync(fs, remotePath); + await Storage.DownloadStreamAsync(fs, remotePath); } } diff --git a/Parallel.Core/Settings/FileSystemCredentials.cs b/Parallel.Core/Settings/FileSystemCredentials.cs index 8000bf9..ac7d4f9 100644 --- a/Parallel.Core/Settings/FileSystemCredentials.cs +++ b/Parallel.Core/Settings/FileSystemCredentials.cs @@ -7,7 +7,7 @@ namespace Parallel.Core.Settings { /// - /// Represents credentials used to gain access to various s. + /// Represents credentials used to gain access to various s. /// public class FileSystemCredentials { diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs index 76f5652..82b0077 100644 --- a/Parallel.Core/Settings/LocalVaultConfig.cs +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -21,7 +21,7 @@ public class LocalVaultConfig public string Name { get; set; } /// - /// The credentials needed to log in to the associated . + /// The credentials needed to log in to the associated . /// public FileSystemCredentials FileSystem { get; } diff --git a/Parallel.Core/IO/FileSystem/IFileSystem.cs b/Parallel.Core/Storage/IStorageProvider.cs similarity index 95% rename from Parallel.Core/IO/FileSystem/IFileSystem.cs rename to Parallel.Core/Storage/IStorageProvider.cs index 74e6f8e..7c6ce3f 100644 --- a/Parallel.Core/IO/FileSystem/IFileSystem.cs +++ b/Parallel.Core/Storage/IStorageProvider.cs @@ -14,7 +14,7 @@ namespace Parallel.Core.IO.FileSystem /// /// Defines the way for communicating with a file system. /// - public interface IFileSystem : IDisposable + public interface IStorageProvider : IDisposable { /// /// Creates all directories and subdirectories in the specified path unless they already exist. @@ -67,6 +67,7 @@ public interface IFileSystem : IDisposable /// /// /// + /// The size, in bytes, of the uploaded stream. Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress); /// diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/Storage/LocalStorageProvider.cs similarity index 88% rename from Parallel.Core/IO/FileSystem/DotNetFileSystem.cs rename to Parallel.Core/Storage/LocalStorageProvider.cs index 31035fc..15f87e1 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/Storage/LocalStorageProvider.cs @@ -16,15 +16,15 @@ namespace Parallel.Core.IO.FileSystem /// /// Represents the wrapper for a default dotnet file system. /// - public class DotNetFileSystem : IFileSystem + public class LocalStorageProvider : IStorageProvider { private readonly LocalVaultConfig _vaultConfig; /// - /// Represents an for interacting with physical machine hardware. + /// Represents an for interacting with physical machine hardware. /// /// The vault to use. - public DotNetFileSystem(LocalVaultConfig vaultConfig) + public LocalStorageProvider(LocalVaultConfig vaultConfig) { _vaultConfig = vaultConfig; } @@ -49,12 +49,9 @@ public Task DeleteDirectoryAsync(string path) /// public Task DeleteFileAsync(string path) { - if (File.Exists(path)) - { - File.SetAttributes(path, ~FileAttributes.ReadOnly & File.GetAttributes(path)); - Microsoft.VisualBasic.FileIO.FileSystem.DeleteFile(path, UIOption.OnlyErrorDialogs, RecycleOption.SendToRecycleBin); - } - + if (!File.Exists(path)) return Task.CompletedTask; + File.SetAttributes(path, ~FileAttributes.ReadOnly & File.GetAttributes(path)); + File.Delete(path); return Task.CompletedTask; } @@ -131,7 +128,7 @@ public async Task UploadStreamAsync(Stream input, string remotePath) await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); await input.CopyToAsync(gzipStream); - //File.SetAttributes(remotePath, File.GetAttributes(remotePath) | FileAttributes.ReadOnly); + File.SetAttributes(remotePath, File.GetAttributes(remotePath) | FileAttributes.ReadOnly); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs b/Parallel.Core/Storage/SshStorageProvider.cs similarity index 80% rename from Parallel.Core/IO/FileSystem/SftpFileSystem.cs rename to Parallel.Core/Storage/SshStorageProvider.cs index 9841c27..28847c2 100644 --- a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs +++ b/Parallel.Core/Storage/SshStorageProvider.cs @@ -17,16 +17,16 @@ namespace Parallel.Core.IO.FileSystem /// /// Represents the wrapper for an SFTP file system through SSH. /// - public class SftpFileSystem : IFileSystem + public class SshStorageProvider : IStorageProvider { private readonly ConnectionInfo _connectionInfo; private readonly SftpClient _client; /// - /// Represents an for interacting with an SSH server. + /// Represents an for interacting with an SSH server. /// /// The credentials to log in with. - public SftpFileSystem(LocalVaultConfig localVault) + public SshStorageProvider(LocalVaultConfig localVault) { _connectionInfo = new ConnectionInfo(localVault.FileSystem.Address, localVault.FileSystem.Username, new PasswordAuthenticationMethod(localVault.FileSystem.Username, Encryption.Decode(localVault.FileSystem.Password))); _client = new SftpClient(_connectionInfo); @@ -144,31 +144,10 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres public async Task UploadStreamAsync(Stream input, string remotePath) { await using SftpFileStream createStream = _client.Create(remotePath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await using GZipStream gzipStream = new(createStream, CompressionLevel.SmallestSize); await input.CopyToAsync(gzipStream); - //_client.ChangePermissions(remotePath, 444); - } - - /// - public async Task UploadFileAsync(string sourcePath, string destinationPath) - { - if (await ExistsAsync(destinationPath)) _client.ChangePermissions(destinationPath, 644); - - string parentDir = string.Empty; - foreach (string subPath in destinationPath.Split('/')) - { - parentDir += $"/{subPath}"; - if (!await _client.ExistsAsync(parentDir)) - { - await _client.CreateDirectoryAsync(parentDir); - } - } - await using SftpFileStream createStream = _client.Create(destinationPath); - await using FileStream openStream = File.OpenRead(sourcePath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream); - _client.ChangePermissions(destinationPath, 444); + _client.ChangePermissions(remotePath, 444); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/FileSystemManager.cs b/Parallel.Core/Storage/StorageProvider.cs similarity index 81% rename from Parallel.Core/IO/FileSystem/FileSystemManager.cs rename to Parallel.Core/Storage/StorageProvider.cs index adb410f..9ad1f64 100644 --- a/Parallel.Core/IO/FileSystem/FileSystemManager.cs +++ b/Parallel.Core/Storage/StorageProvider.cs @@ -28,18 +28,18 @@ public enum FileService /// /// Represents the way to connect to different file system associations. This class cannot be inherited. /// - public static class FileSystemManager + public static class StorageProvider { /// /// Creates a new file system association. /// /// The vault needed for the associated file system. - public static IFileSystem CreateNew(LocalVaultConfig vaultConfig) + public static IStorageProvider CreateNew(LocalVaultConfig vaultConfig) { return vaultConfig.FileSystem.Service switch { - FileService.Local => new DotNetFileSystem(vaultConfig), - FileService.Remote => new SftpFileSystem(vaultConfig), + FileService.Local => new LocalStorageProvider(vaultConfig), + FileService.Remote => new SshStorageProvider(vaultConfig), //FileService.Cloud => new AmazonS3FileSystem(credentials), _ => null }; diff --git a/Parallel.Core/Utils/Formatter.cs b/Parallel.Core/Utils/Formatter.cs index 411d22d..daed03b 100644 --- a/Parallel.Core/Utils/Formatter.cs +++ b/Parallel.Core/Utils/Formatter.cs @@ -12,11 +12,11 @@ public class Formatter /// /// The bytes to convert. /// The bytes formatted as a string. - public static string FromBytes(long bytes) + public static string FromBytes(double bytes) { string[] sizeSuffixes = { "Bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB" }; int sizeIndex = 0; - double size = bytes; + double size = Math.Abs(bytes); // use absolute for scaling while (size >= 1000 && sizeIndex < sizeSuffixes.Length - 1) { @@ -24,9 +24,12 @@ public static string FromBytes(long bytes) size /= 1000; } - return $"{size:N2} {sizeSuffixes[sizeIndex]}"; + // add the sign back + string sign = bytes < 0 ? "-" : ""; + return $"{sign}{size:N2} {sizeSuffixes[sizeIndex]}"; } + /// /// Formats a with the corresponding data volume. /// From c54c59836fc188b2a5bbfd24ec5d95173a333e60 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Wed, 10 Dec 2025 01:02:03 -0600 Subject: [PATCH 05/17] Added enabled/disable option --- Parallel.Core/Settings/LocalVaultConfig.cs | 5 +++++ Parallel.Core/Settings/ParallelConfig.cs | 14 +++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs index 82b0077..82d8203 100644 --- a/Parallel.Core/Settings/LocalVaultConfig.cs +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -20,6 +20,11 @@ public class LocalVaultConfig /// public string Name { get; set; } + /// + /// If the current vault config is enabled. + /// + public bool Enabled { get; set; } = true; + /// /// The credentials needed to log in to the associated . /// diff --git a/Parallel.Core/Settings/ParallelConfig.cs b/Parallel.Core/Settings/ParallelConfig.cs index 17d382a..dc95650 100644 --- a/Parallel.Core/Settings/ParallelConfig.cs +++ b/Parallel.Core/Settings/ParallelConfig.cs @@ -26,6 +26,8 @@ public class ParallelConfig MaxDegreeOfParallelism = Load().MaxConcurrentProcesses }; + public static int MaxUploads { get; } = Load().MaxConcurrentUploads; + // /// // /// The address that will accept incoming commands. // /// Default: 127.0.0.1 @@ -46,9 +48,15 @@ public class ParallelConfig /// /// Gets or sets the maximum number of concurrent processes that can run. - /// Default: Half the processor count. + /// Default: The processor count. + /// + public int MaxConcurrentProcesses { get; set; } = Environment.ProcessorCount; + + /// + /// Gets or sets the maximum number of concurrent processes that can run. + /// Default: 4 /// - public int MaxConcurrentProcesses { get; set; } = Math.Clamp(Environment.ProcessorCount / 2, 1, Environment.ProcessorCount); + public int MaxConcurrentUploads { get; set; } = 4; /// /// The amount of time, in days, to hold a file before it can be cleaned. @@ -118,7 +126,7 @@ public async Task ForEachVaultAsync(Func actionAsync, Ca CancellationToken = cancellationToken }; - await System.Threading.Tasks.Parallel.ForEachAsync(Vaults, options, async (vault, ct) => + await System.Threading.Tasks.Parallel.ForEachAsync(Vaults.Where(v => v.Enabled), options, async (vault, ct) => { await actionAsync(vault); }); From 72661de02963710691d7d33f1af32fbd67f9b69f Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Wed, 10 Dec 2025 07:00:17 -0600 Subject: [PATCH 06/17] Uploads are stupid fast now lol --- Parallel.Cli/Commands/CleanCommand.cs | 1 + Parallel.Cli/Commands/PushCommand.cs | 9 +- Parallel.Cli/Parallel.Cli.csproj | 4 - Parallel.Cli/Utils/ProgressBarReporter.cs | 68 +++++++ .../Database/Contexts/SqliteContext.cs | 6 +- Parallel.Core/IO/Scanning/FileScanner.cs | 84 ++++----- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 169 ++++++++++++++---- Parallel.Core/Settings/ParallelConfig.cs | 10 +- Parallel.Core/Storage/LocalStorageProvider.cs | 4 +- Parallel.Core/Utils/TransferWorker.cs | 20 +++ 10 files changed, 287 insertions(+), 88 deletions(-) create mode 100644 Parallel.Cli/Utils/ProgressBarReporter.cs create mode 100644 Parallel.Core/Utils/TransferWorker.cs diff --git a/Parallel.Cli/Commands/CleanCommand.cs b/Parallel.Cli/Commands/CleanCommand.cs index b05a096..57bb19d 100644 --- a/Parallel.Cli/Commands/CleanCommand.cs +++ b/Parallel.Cli/Commands/CleanCommand.cs @@ -1,6 +1,7 @@ // Copyright 2025 Kyle Ebbinga using System.CommandLine; +using System.Runtime.InteropServices; using Parallel.Cli.Utils; using Parallel.Core.IO.Scanning; using Parallel.Core.IO.Syncing; diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 6668fcb..6c40746 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -1,6 +1,7 @@ // Copyright 2025 Kyle Ebbinga using System.CommandLine; +using System.Diagnostics; using Parallel.Cli.Utils; using Parallel.Core.Diagnostics; using Parallel.Core.IO; @@ -13,6 +14,8 @@ namespace Parallel.Cli.Commands { public class PushCommand : Command { + private Stopwatch _sw = new Stopwatch(); + private Command addCmd = new("add", "Adds a new directory to the sync list."); private Command listCmd = new("list", "Shows all directories in the sync list."); private Command removeCmd = new("remove", "Removes a directory from the sync list."); @@ -28,6 +31,7 @@ public PushCommand() : base("push", "Pushes changed files to vaults.") this.AddOption(_verboseOpt); this.SetHandler(async (path, config, verbose) => { + _sw = Stopwatch.StartNew(); if (string.IsNullOrEmpty(path)) { await SyncSystemAsync(); @@ -90,9 +94,10 @@ await Program.Settings.ForEachVaultAsync(async vault => CommandLine.WriteLine(vault, $"Backing up {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); await syncManager.PushFilesAsync(files, new ProgressReport(vault, successFiles)); - await syncManager.DisconnectAsync(); + //await syncManager.PushFilesAsync(files, new ProgressBarReporter()); + //await syncManager.DisconnectAsync(); - CommandLine.WriteLine(vault, $"Successfully pushed {successFiles.ToString("N0")} files to '{vault.FileSystem.RootDirectory}'.", ConsoleColor.Green); + CommandLine.WriteLine(vault, $"Successfully pushed {successFiles.ToString("N0")} files in {_sw.Elapsed}.", ConsoleColor.Green); }); } } diff --git a/Parallel.Cli/Parallel.Cli.csproj b/Parallel.Cli/Parallel.Cli.csproj index fb20e4e..7c0c6b5 100644 --- a/Parallel.Cli/Parallel.Cli.csproj +++ b/Parallel.Cli/Parallel.Cli.csproj @@ -50,8 +50,4 @@ - - - - diff --git a/Parallel.Cli/Utils/ProgressBarReporter.cs b/Parallel.Cli/Utils/ProgressBarReporter.cs new file mode 100644 index 0000000..7c15f60 --- /dev/null +++ b/Parallel.Cli/Utils/ProgressBarReporter.cs @@ -0,0 +1,68 @@ +// Copyright 2025 Kyle Ebbinga + +using System.Collections.Concurrent; +using Parallel.Core.Diagnostics; +using Parallel.Core.Models; + +namespace Parallel.Cli.Utils +{ + public class ProgressBarReporter : IProgressReporter + { + private readonly ConcurrentDictionary _lines = new(); + private readonly object _consoleLock = new(); + private int _nextLine = 0; + + // Assign a line for a file + private int GetLine(string fileId) + { + if (!_lines.ContainsKey(fileId)) + { + _lines[fileId] = (_nextLine++).ToString(); + } + + return int.Parse(_lines[fileId]); + } + + public void Report(ProgressOperation operation, SystemFile file) + { + int line = GetLine(file.Id); + string msg = $"{operation,-10} {file.Name,-30} "; + + // Optionally show progress, e.g., percentage + if (file.LocalSize > 0 && file.RemoteSize > 0) + { + double percent = (double)file.RemoteSize / file.LocalSize * 100; + msg += $"{percent:0.0}%"; + } + + lock (_consoleLock) + { + Console.SetCursorPosition(0, line); + Console.Write(msg.PadRight(Console.WindowWidth)); + } + } + + public void Reset() + { + lock (_consoleLock) + { + Console.Clear(); + _lines.Clear(); + _nextLine = 0; + } + } + + + public void Failed(Exception exception, SystemFile file) + { + int line = GetLine(file.Id); + lock (_consoleLock) + { + Console.SetCursorPosition(0, line); + Console.ForegroundColor = ConsoleColor.Red; + Console.Write($"FAILED {file.Name}: {exception.Message}".PadRight(Console.WindowWidth)); + Console.ResetColor(); + } + } + } +} \ No newline at end of file diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index f3292cc..9f4e133 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -67,6 +67,7 @@ public async Task RemoveFileAsync(SystemFile file) await connection.ExecuteAsync(sql, new { file.Id }); } + /// public async Task GetLocalSizeAsync() { using IDbConnection connection = CreateConnection(); @@ -81,6 +82,7 @@ public async Task GetRemoteSizeAsync() return await connection.QuerySingleOrDefaultAsync(sql); } + /// public async Task GetTotalFilesAsync(bool deleted) { using IDbConnection connection = CreateConnection(); @@ -100,8 +102,8 @@ public async Task> GetFilesAsync(string path) public async Task> GetFilesAsync(string path, bool deleted) { using IDbConnection connection = CreateConnection(); - string sql = $"SELECT * FROM files WHERE deleted = @deleted ORDER BY lastupdate DESC"; - return await connection.QueryAsync(sql,new { deleted }); + string sql = $"SELECT * FROM files WHERE localpath LIKE @Path AND deleted = @deleted ORDER BY lastupdate DESC"; + return await connection.QueryAsync(sql, new { Path = $"%{path}%", deleted }); } /// diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index cdd0ac2..4916b34 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -60,7 +60,10 @@ public async Task GetFileChangesAsync(string path, string[] ignore ConcurrentBag scannedFiles = new(); ConcurrentBag changedFiles = new(); + Log.Debug("Getting system files..."); HashSet localFiles = FileScanner.GetFiles(path, ignoreFolders, ".").ToHashSet(); + + Log.Debug("Getting database files..."); IEnumerable remoteFiles = await _db.GetFilesAsync(path, false); System.Threading.Tasks.Parallel.ForEach(remoteFiles, ParallelConfig.Options, (remoteFile, ct) => { @@ -69,14 +72,14 @@ public async Task GetFileChangesAsync(string path, string[] ignore SystemFile localFile = new SystemFile(remoteFile.LocalPath); if (IsIgnored(localFile.LocalPath, ignoreFolders)) { - //Log.Debug($"Ignored -> {localFile.LocalPath}"); + Log.Debug($"Ignored -> {localFile.LocalPath}"); localFile.RemotePath = remoteFile.RemotePath; localFile.Deleted = true; changedFiles.Add(localFile); } else if (HasChanged(localFile, remoteFile)) { - //Log.Debug($"Changed -> {localFile.LocalPath}"); + Log.Debug($"Changed -> {localFile.LocalPath}"); localFile.RemotePath = remoteFile.RemotePath; changedFiles.Add(localFile); } @@ -85,7 +88,7 @@ public async Task GetFileChangesAsync(string path, string[] ignore } else { - //Log.Debug($"Deleted -> {remoteFile.LocalPath}"); + Log.Debug($"Deleted -> {remoteFile.LocalPath}"); remoteFile.Deleted = true; changedFiles.Add(remoteFile); } @@ -93,16 +96,24 @@ public async Task GetFileChangesAsync(string path, string[] ignore HashSet remainingFiles = localFiles.Except(new HashSet(scannedFiles)).ToHashSet(); Log.Debug($"{remainingFiles.Count} files are untracked! Adding..."); - System.Threading.Tasks.Parallel.ForEach(remainingFiles, ParallelConfig.Options, (file, ct) => + + remainingFiles.AsParallel().WithDegreeOfParallelism(Environment.ProcessorCount).Where(f => !IsIgnored(f, ignoreFolders)).ForAll(file => { - if (!IsIgnored(file, ignoreFolders)) - { - //Log.Debug($"Created -> {file}"); - changedFiles.Add(new SystemFile(file) { RemotePath = PathBuilder.Remote(file, _config) }); - } + Log.Debug($"Created -> {file}"); + changedFiles.Add(new SystemFile(file)); }); - Log.Debug($"{localFiles.Count} files remaining."); + // System.Threading.Tasks.Parallel.ForEach(remainingFiles, ParallelConfig.Options, (file, ct) => + // { + // if (!IsIgnored(file, ignoreFolders)) + // { + // Log.Debug($"Created -> {file}"); + // changedFiles.Add(new SystemFile(file) { RemotePath = PathBuilder.Remote(file, _config) }); + // } + // + // Log.Debug($"Done"); + // }); + Log.Information($"Found {changedFiles.Count:N0} changes in '{path}'"); return changedFiles.ToArray(); } @@ -322,44 +333,33 @@ public static bool IsIgnored(string path, string[] exempt) { foreach (string entry in exempt) { - if (!_cache.TryGetValue(entry, out Regex? regex)) + if (path.StartsWith(entry)) { - regex = BuildRegexCache(entry); - _cache[entry] = regex; - } - - if (regex.IsMatch(path)) return true; - } - - return false; - } - - private static Regex BuildRegexCache(string entry) - { - string pattern; - if (!entry.Contains('*') && !entry.EndsWith("/") && !entry.EndsWith("\\")) - { - pattern = "^" + Regex.Escape(entry); - return new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); - } + } - if (entry.EndsWith("/") || entry.EndsWith("\\")) - { - string folder = entry.TrimEnd('/', '\\'); - pattern = @"(^|\\)" + Regex.Escape(folder) + @"(\\|$)"; - return new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); - } + if (entry.EndsWith('/')) + { + string[] folders = path.Split('\\'); + foreach (string dir in folders) + { + if (dir.ToLower() == entry.Remove(entry.Length - 1, 1).ToLower()) + { + return true; + } + } + } - if (entry.StartsWith("*")) - { - string ext = Regex.Escape(entry.TrimStart('*')); - pattern = ".*" + ext + "$"; - return new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); + if (entry.StartsWith('*')) + { + if (path.EndsWith(entry.Replace("*", string.Empty))) + { + return true; + } + } } - pattern = "^" + Regex.Escape(entry) + "$"; - return new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); + return false; } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index 7d464cb..2c86d52 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -1,7 +1,10 @@ // Copyright 2025 Kyle Ebbinga +using System.Buffers; using System.Collections.Concurrent; +using System.Diagnostics; using System.Reflection.Metadata; +using System.Threading.Channels; using Newtonsoft.Json.Linq; using Parallel.Core.Database; using Parallel.Core.Diagnostics; @@ -9,6 +12,7 @@ using Parallel.Core.Models; using Parallel.Core.Security; using Parallel.Core.Settings; +using Parallel.Core.Utils; namespace Parallel.Core.IO.Syncing { @@ -33,52 +37,155 @@ public ObjectSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) { - await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + TimeSpan uploadTimeout = TimeSpan.FromSeconds(30); + TimeSpan writeBlockWarn = TimeSpan.FromSeconds(5); + long enqueued = 0, dequeued = 0, uploadStarted = 0, uploadDone = 0; + Channel channel = Channel.CreateBounded(new BoundedChannelOptions(256) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = false, + SingleWriter = false + }); + + // WORKERS + Task[] workerTasks = Enumerable.Range(0, ParallelConfig.MaxTransfers).Select(workerId => Task.Run(async () => { + Log.Debug($"[WORKER {workerId}] START"); try { - if (file.Deleted) + await foreach (TransferWorker job in channel.Reader.ReadAllAsync()) { - progress.Report(ProgressOperation.Archiving, file); - await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); - await Database.AddFileAsync(file); + Interlocked.Increment(ref dequeued); + Log.Debug($"[WORKER {workerId}] DEQUEUE -> {job.RemotePath} dequeued={dequeued}"); + + try + { + Interlocked.Increment(ref uploadStarted); + Log.Debug($"[WORKER {workerId}] UPLOAD START -> {job.RemotePath} started={uploadStarted}"); + + string fullPath = PathBuilder.Combine(job.RemotePath, job.Filename); + if (await Storage.ExistsAsync(fullPath)) + { + Log.Debug($"[WORKER {workerId}] UPLOAD SKIPPED -> {job.RemotePath} done={uploadDone}"); + continue; + } + + if (!await Storage.ExistsAsync(job.RemotePath)) + await Storage.CreateDirectoryAsync(job.RemotePath); + + await using MemoryStream ms = new MemoryStream(job.Data, writable: false); + + using CancellationTokenSource cts = new CancellationTokenSource(uploadTimeout); + Task uploadTask = Storage.UploadStreamAsync(ms, fullPath); + await uploadTask; + + Interlocked.Increment(ref uploadDone); + Log.Debug($"[WORKER {workerId}] UPLOAD DONE -> {job.RemotePath} done={uploadDone}"); + } + catch (Exception wex) + { + Log.Error($"[WORKER {workerId}] UPLOAD EX -> {job.RemotePath} : {wex}"); + job.OnError?.Invoke(wex); + } } - else - { - int bytesRead = 0; - byte[] buffer = new byte[ChunkSize]; - await using FileStream fs = File.OpenRead(file.LocalPath); - int index = 0; - progress.Report(ProgressOperation.Pushing, file); - await Database.AddFileAsync(file); + Log.Debug($"[WORKER {workerId}] READER COMPLETED"); + } + finally + { + Log.Debug($"[WORKER {workerId}] EXIT"); + } + })).ToArray(); - while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) + // PRODUCER + var producer = Task.Run(async () => + { + Log.Debug("PRODUCER START"); + try + { + await System.Threading.Tasks.Parallel.ForEachAsync(files, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, async (file, token) => + { + try { - await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); - string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); - await Database.AddObjectAsync(file.Id, hash, index); - index++; - - string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); - string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); - string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - if (!await Storage.ExistsAsync(remotePath)) + if (file.Deleted) { - if (!await Storage.ExistsAsync(parentDir)) await Storage.CreateDirectoryAsync(parentDir); - await Storage.UploadStreamAsync(ms, remotePath); + progress.Report(ProgressOperation.Archiving, file); + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); + await Database.AddFileAsync(file); + return; } - } - await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pushed); - } + progress.Report(ProgressOperation.Pushing, file); + await Database.AddFileAsync(file); + + await using FileStream fs = new FileStream(file.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: ChunkSize, useAsync: true); + + byte[] buffer = ArrayPool.Shared.Rent(ChunkSize); + try + { + int index = 0; + int bytesRead; + while ((bytesRead = await fs.ReadAsync(buffer.AsMemory(0, ChunkSize), token)) > 0) + { + byte[] chunk = new byte[bytesRead]; + Buffer.BlockCopy(buffer, 0, chunk, 0, bytesRead); + + string hash = HashGenerator.CreateSHA256(chunk); + await Database.AddObjectAsync(file.Id, hash, index++); + + string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); + string remotePath = PathBuilder.Combine(parentDir, hash[4..]); + + TransferWorker job = new TransferWorker(chunk, parentDir, hash[4..], ex => progress.Failed(ex, file)); + + Task writeTask = channel.Writer.WriteAsync(job, token).AsTask(); + Task winner = await Task.WhenAny(writeTask, Task.Delay(writeBlockWarn, token)); + if (winner != writeTask) + { + Log.Warning($"PRODUCER: channel.Writer.WriteAsync is blocked > {writeBlockWarn} (remote={remotePath}, enq={enqueued}, deq={dequeued})"); + await writeTask; + } + + Interlocked.Increment(ref enqueued); + Log.Debug($"ENQUEUE -> {remotePath}. enqueued={enqueued}"); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pushed); + } + catch (Exception pex) + { + Log.Error($"PRODUCER EX processing {file.LocalPath}: {pex}"); + progress.Failed(pex, file); + } + }); } - catch (Exception ex) + finally { - Log.Error(ex.GetBaseException().ToString()); - progress.Failed(ex, file); + Log.Debug("PRODUCER COMPLETE -> Completing writer"); + channel.Writer.Complete(); } }); + + // MONITOR + Task monitor = Task.Run(async () => + { + Stopwatch sw = Stopwatch.StartNew(); + while (!Task.WhenAll(workerTasks).IsCompleted) + { + Log.Debug($"PIPELINE STATS @ {sw.Elapsed}: enq={enqueued}, deq={dequeued}, upStarted={uploadStarted}, upDone={uploadDone}"); + await Task.Delay(1000); + } + }); + + await Task.WhenAll(producer, monitor).ConfigureAwait(false); + await Task.WhenAll(workerTasks).ConfigureAwait(false); + Log.Debug("PUSHFILES FINISHED"); } /// diff --git a/Parallel.Core/Settings/ParallelConfig.cs b/Parallel.Core/Settings/ParallelConfig.cs index dc95650..54038c4 100644 --- a/Parallel.Core/Settings/ParallelConfig.cs +++ b/Parallel.Core/Settings/ParallelConfig.cs @@ -26,7 +26,7 @@ public class ParallelConfig MaxDegreeOfParallelism = Load().MaxConcurrentProcesses }; - public static int MaxUploads { get; } = Load().MaxConcurrentUploads; + public static int MaxTransfers { get; } = Load().MaxConcurrentTransfers; // /// // /// The address that will accept incoming commands. @@ -42,9 +42,9 @@ public class ParallelConfig /// /// Gets or sets the maximum number of concurrent vaults that can run. - /// Default: 2 + /// Default: 1 /// - public int MaxConcurrentVaults { get; set; } = 2; + public int MaxConcurrentVaults { get; set; } = 1; /// /// Gets or sets the maximum number of concurrent processes that can run. @@ -53,10 +53,10 @@ public class ParallelConfig public int MaxConcurrentProcesses { get; set; } = Environment.ProcessorCount; /// - /// Gets or sets the maximum number of concurrent processes that can run. + /// Gets or sets the maximum number of concurrent transfers that can run. /// Default: 4 /// - public int MaxConcurrentUploads { get; set; } = 4; + public int MaxConcurrentTransfers { get; set; } = 4; /// /// The amount of time, in days, to hold a file before it can be cleaned. diff --git a/Parallel.Core/Storage/LocalStorageProvider.cs b/Parallel.Core/Storage/LocalStorageProvider.cs index 15f87e1..3fbfe18 100644 --- a/Parallel.Core/Storage/LocalStorageProvider.cs +++ b/Parallel.Core/Storage/LocalStorageProvider.cs @@ -124,11 +124,11 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options /// public async Task UploadStreamAsync(Stream input, string remotePath) { - await using FileStream createStream = File.Create(remotePath); + await using FileStream createStream = File.OpenWrite(remotePath); await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); await input.CopyToAsync(gzipStream); - File.SetAttributes(remotePath, File.GetAttributes(remotePath) | FileAttributes.ReadOnly); + //File.SetAttributes(remotePath, File.GetAttributes(remotePath) | FileAttributes.ReadOnly); } } } \ No newline at end of file diff --git a/Parallel.Core/Utils/TransferWorker.cs b/Parallel.Core/Utils/TransferWorker.cs new file mode 100644 index 0000000..301f5c1 --- /dev/null +++ b/Parallel.Core/Utils/TransferWorker.cs @@ -0,0 +1,20 @@ +// Copyright 2025 Kyle Ebbinga + +namespace Parallel.Core.Utils +{ + public class TransferWorker + { + public byte[] Data { get; } + public string RemotePath { get; } + public string Filename { get; } + public Action? OnError { get; } + + public TransferWorker(byte[] data, string remotePath, string filename, Action? onError = null) + { + Data = data; + RemotePath = remotePath; + Filename = filename; + OnError = onError; + } + } +} \ No newline at end of file From f08310dbdc33489ee3d2cdfcfcc896379195257b Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Thu, 11 Dec 2025 02:10:09 -0600 Subject: [PATCH 07/17] Better push logging --- Parallel.Cli/Commands/PushCommand.cs | 2 +- Parallel.Cli/Utils/CommandLine.cs | 4 +- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 106 ++++++++---------- 3 files changed, 50 insertions(+), 62 deletions(-) diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 6c40746..2f125de 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -95,7 +95,7 @@ await Program.Settings.ForEachVaultAsync(async vault => CommandLine.WriteLine(vault, $"Backing up {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); await syncManager.PushFilesAsync(files, new ProgressReport(vault, successFiles)); //await syncManager.PushFilesAsync(files, new ProgressBarReporter()); - //await syncManager.DisconnectAsync(); + await syncManager.DisconnectAsync(); CommandLine.WriteLine(vault, $"Successfully pushed {successFiles.ToString("N0")} files in {_sw.Elapsed}.", ConsoleColor.Green); }); diff --git a/Parallel.Cli/Utils/CommandLine.cs b/Parallel.Cli/Utils/CommandLine.cs index 0e01b11..b11f710 100644 --- a/Parallel.Cli/Utils/CommandLine.cs +++ b/Parallel.Cli/Utils/CommandLine.cs @@ -104,7 +104,7 @@ public static void WriteLine(LocalVaultConfig localVault, object value, ConsoleC public static void WriteLine(object value, ConsoleColor color = ConsoleColor.Gray) { string baseLog = $"{value}"; - switch(color) + /*switch(color) { default: Log.Information(baseLog); @@ -117,7 +117,7 @@ public static void WriteLine(object value, ConsoleColor color = ConsoleColor.Gra case ConsoleColor.Red: Log.Error(baseLog); break; - } + }*/ lock (_consoleLock) { diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index 2c86d52..c389f5b 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -37,9 +37,10 @@ public ObjectSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) { + long queued = 0, completed = 0, total = 0; TimeSpan uploadTimeout = TimeSpan.FromSeconds(30); TimeSpan writeBlockWarn = TimeSpan.FromSeconds(5); - long enqueued = 0, dequeued = 0, uploadStarted = 0, uploadDone = 0; + Channel channel = Channel.CreateBounded(new BoundedChannelOptions(256) { FullMode = BoundedChannelFullMode.Wait, @@ -47,63 +48,53 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter SingleWriter = false }); - // WORKERS Task[] workerTasks = Enumerable.Range(0, ParallelConfig.MaxTransfers).Select(workerId => Task.Run(async () => { - Log.Debug($"[WORKER {workerId}] START"); - try + await foreach (TransferWorker job in channel.Reader.ReadAllAsync()) { - await foreach (TransferWorker job in channel.Reader.ReadAllAsync()) - { - Interlocked.Increment(ref dequeued); - Log.Debug($"[WORKER {workerId}] DEQUEUE -> {job.RemotePath} dequeued={dequeued}"); + Interlocked.Decrement(ref queued); - try + try + { + Interlocked.Increment(ref total); + string fullPath = PathBuilder.Combine(job.RemotePath, job.Filename); + if (await Storage.ExistsAsync(fullPath)) { - Interlocked.Increment(ref uploadStarted); - Log.Debug($"[WORKER {workerId}] UPLOAD START -> {job.RemotePath} started={uploadStarted}"); - - string fullPath = PathBuilder.Combine(job.RemotePath, job.Filename); - if (await Storage.ExistsAsync(fullPath)) - { - Log.Debug($"[WORKER {workerId}] UPLOAD SKIPPED -> {job.RemotePath} done={uploadDone}"); - continue; - } - - if (!await Storage.ExistsAsync(job.RemotePath)) - await Storage.CreateDirectoryAsync(job.RemotePath); + Log.Debug($"[WORKER {workerId}] UPLOAD SKIPPED: {fullPath}"); + continue; + } - await using MemoryStream ms = new MemoryStream(job.Data, writable: false); + if (!await Storage.ExistsAsync(job.RemotePath)) await Storage.CreateDirectoryAsync(job.RemotePath); + await using MemoryStream ms = new MemoryStream(job.Data, false); - using CancellationTokenSource cts = new CancellationTokenSource(uploadTimeout); - Task uploadTask = Storage.UploadStreamAsync(ms, fullPath); - await uploadTask; + using CancellationTokenSource cts = new CancellationTokenSource(uploadTimeout); + Task uploadTask = Storage.UploadStreamAsync(ms, fullPath); - Interlocked.Increment(ref uploadDone); - Log.Debug($"[WORKER {workerId}] UPLOAD DONE -> {job.RemotePath} done={uploadDone}"); - } - catch (Exception wex) + Task timeoutTask = await Task.WhenAny(uploadTask, Task.Delay(uploadTimeout, cts.Token)); + if (timeoutTask != uploadTask) { - Log.Error($"[WORKER {workerId}] UPLOAD EX -> {job.RemotePath} : {wex}"); - job.OnError?.Invoke(wex); + Log.Error($"[WORKER {workerId}] UPLOAD TIMEOUT: {fullPath}"); + job.OnError?.Invoke(new TimeoutException($"Upload timed out after {uploadTimeout}")); + continue; } - } - Log.Debug($"[WORKER {workerId}] READER COMPLETED"); - } - finally - { - Log.Debug($"[WORKER {workerId}] EXIT"); + await uploadTask; + Interlocked.Increment(ref completed); + Log.Debug($"[WORKER {workerId}] UPLOAD COMPLETE: {fullPath} complete={completed}"); + } + catch (Exception ex) + { + Log.Error($"[WORKER {workerId}] UPLOAD ERROR: {job.Filename} : {ex}"); + job.OnError?.Invoke(ex); + } } })).ToArray(); - // PRODUCER - var producer = Task.Run(async () => + Task producer = Task.Run(async () => { - Log.Debug("PRODUCER START"); try { - await System.Threading.Tasks.Parallel.ForEachAsync(files, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, async (file, token) => + await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { try { @@ -119,13 +110,12 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter await Database.AddFileAsync(file); await using FileStream fs = new FileStream(file.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: ChunkSize, useAsync: true); - byte[] buffer = ArrayPool.Shared.Rent(ChunkSize); + try { - int index = 0; - int bytesRead; - while ((bytesRead = await fs.ReadAsync(buffer.AsMemory(0, ChunkSize), token)) > 0) + int index = 0, bytesRead; + while ((bytesRead = await fs.ReadAsync(buffer.AsMemory(0, ChunkSize), ct)) > 0) { byte[] chunk = new byte[bytesRead]; Buffer.BlockCopy(buffer, 0, chunk, 0, bytesRead); @@ -137,18 +127,17 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - TransferWorker job = new TransferWorker(chunk, parentDir, hash[4..], ex => progress.Failed(ex, file)); + TransferWorker job = new(chunk, parentDir, hash[4..], ex => progress.Failed(ex, file)); - Task writeTask = channel.Writer.WriteAsync(job, token).AsTask(); - Task winner = await Task.WhenAny(writeTask, Task.Delay(writeBlockWarn, token)); - if (winner != writeTask) + Task writeTask = channel.Writer.WriteAsync(job, ct).AsTask(); + Task timeoutTask = await Task.WhenAny(writeTask, Task.Delay(writeBlockWarn, ct)); + if (timeoutTask != writeTask) { - Log.Warning($"PRODUCER: channel.Writer.WriteAsync is blocked > {writeBlockWarn} (remote={remotePath}, enq={enqueued}, deq={dequeued})"); + Log.Warning($"[PRODUCER] channel.Writer.WriteAsync is blocked > {writeBlockWarn} (remote={remotePath})"); await writeTask; } - Interlocked.Increment(ref enqueued); - Log.Debug($"ENQUEUE -> {remotePath}. enqueued={enqueued}"); + Interlocked.Increment(ref queued); } } finally @@ -158,34 +147,33 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pushed); } - catch (Exception pex) + catch (Exception ex) { - Log.Error($"PRODUCER EX processing {file.LocalPath}: {pex}"); - progress.Failed(pex, file); + Log.Error($"[PRODUCER] ERROR: {file.LocalPath}: {ex}"); + progress.Failed(ex, file); } }); } finally { - Log.Debug("PRODUCER COMPLETE -> Completing writer"); + Log.Debug("[PRODUCER] COMPLETE"); channel.Writer.Complete(); } }); - // MONITOR Task monitor = Task.Run(async () => { Stopwatch sw = Stopwatch.StartNew(); while (!Task.WhenAll(workerTasks).IsCompleted) { - Log.Debug($"PIPELINE STATS @ {sw.Elapsed}: enq={enqueued}, deq={dequeued}, upStarted={uploadStarted}, upDone={uploadDone}"); + Log.Debug($"PIPELINE STATS @ {sw.Elapsed}: queued={queued}, completed={completed}, total={total}"); await Task.Delay(1000); } }); + Log.Debug($"PIPELINE COMPLETED: queued={queued}, completed={completed}, total={total}"); await Task.WhenAll(producer, monitor).ConfigureAwait(false); await Task.WhenAll(workerTasks).ConfigureAwait(false); - Log.Debug("PUSHFILES FINISHED"); } /// From 6135dbcdf0943913e66bc5731d613a973e70c494 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Thu, 11 Dec 2025 03:56:42 -0600 Subject: [PATCH 08/17] Clamped values to prevent 0 process errors --- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 2 +- Parallel.Core/Settings/ParallelConfig.cs | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index c389f5b..a3e5f53 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -48,7 +48,7 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter SingleWriter = false }); - Task[] workerTasks = Enumerable.Range(0, ParallelConfig.MaxTransfers).Select(workerId => Task.Run(async () => + Task[] workerTasks = Enumerable.Range(0, ParallelConfig.MaxStaticTransfers).Select(workerId => Task.Run(async () => { await foreach (TransferWorker job in channel.Reader.ReadAllAsync()) { diff --git a/Parallel.Core/Settings/ParallelConfig.cs b/Parallel.Core/Settings/ParallelConfig.cs index 54038c4..856deae 100644 --- a/Parallel.Core/Settings/ParallelConfig.cs +++ b/Parallel.Core/Settings/ParallelConfig.cs @@ -21,12 +21,18 @@ public class ParallelConfig /// public static string VaultsDir { get; } = Path.Combine(PathBuilder.ProgramData, "Vaults"); + /// + /// Gets a set of static options for . + /// public static ParallelOptions Options { get; } = new ParallelOptions { - MaxDegreeOfParallelism = Load().MaxConcurrentProcesses + MaxDegreeOfParallelism = Math.Max(1, Load().MaxConcurrentProcesses) }; - public static int MaxTransfers { get; } = Load().MaxConcurrentTransfers; + /// + /// Gets the static value for . + /// + public static int MaxStaticTransfers { get; } = Math.Max(1, Load().MaxConcurrentTransfers); // /// // /// The address that will accept incoming commands. From 4771dbe099bbdb40470cc0f09e16a90b3bee8a22 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 00:38:05 -0600 Subject: [PATCH 09/17] Work in progress for optimazing pulls --- Parallel.Cli/Commands/DiskCommand.cs | 8 +-- Parallel.Cli/Commands/PullCommand.cs | 4 +- Parallel.Cli/Commands/VaultsCommand.cs | 2 +- .../Diagnostics/IProgressReporter.cs | 10 ++- Parallel.Core/IO/Blobs/BlobStorage.cs | 1 + Parallel.Core/IO/PathBuilder.cs | 7 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 21 +++--- Parallel.Core/IO/Syncing/FileSyncManager.cs | 11 ++-- Parallel.Core/IO/Syncing/ISyncManager.cs | 3 +- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 64 +++++++++---------- Parallel.Core/Settings/LocalVaultConfig.cs | 13 ++-- Parallel.Core/Settings/RemoteVaultConfig.cs | 6 +- ...emCredentials.cs => StorageCredentials.cs} | 7 +- Parallel.Core/Storage/IStorageProvider.cs | 8 +-- Parallel.Core/Storage/LocalStorageProvider.cs | 8 +-- Parallel.Core/Storage/SshStorageProvider.cs | 13 ++-- ...torageProvider.cs => StorageConnection.cs} | 5 +- Parallel.Core/Utils/TransferWorker.cs | 20 ------ Parallel.Core/Workers/BaseWorker.cs | 33 ++++++++++ Parallel.Core/Workers/DownloadWorker.cs | 14 ++++ Parallel.Core/Workers/FileAggregator.cs | 13 ++++ Parallel.Core/Workers/UploadWorker.cs | 22 +++++++ 22 files changed, 172 insertions(+), 121 deletions(-) rename Parallel.Core/Settings/{FileSystemCredentials.cs => StorageCredentials.cs} (85%) rename Parallel.Core/Storage/{StorageProvider.cs => StorageConnection.cs} (91%) delete mode 100644 Parallel.Core/Utils/TransferWorker.cs create mode 100644 Parallel.Core/Workers/BaseWorker.cs create mode 100644 Parallel.Core/Workers/DownloadWorker.cs create mode 100644 Parallel.Core/Workers/FileAggregator.cs create mode 100644 Parallel.Core/Workers/UploadWorker.cs diff --git a/Parallel.Cli/Commands/DiskCommand.cs b/Parallel.Cli/Commands/DiskCommand.cs index ba713cd..b2372f8 100644 --- a/Parallel.Cli/Commands/DiskCommand.cs +++ b/Parallel.Cli/Commands/DiskCommand.cs @@ -50,17 +50,17 @@ private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) long totalObjects = await db.GetTotalObjectsAsync(); CommandLine.WriteLine($"Using vault '{vault.Name}' ({vault.Id}):"); - CommandLine.WriteLine($"Service Type: {vault.FileSystem.Service}"); - CommandLine.WriteLine($"Root Directory: {vault.FileSystem.RootDirectory}"); + CommandLine.WriteLine($"Service Type: {vault.Credentials.Service}"); + CommandLine.WriteLine($"Root Directory: {vault.Credentials.RootDirectory}"); CommandLine.WriteLine($"Managed Files: {(totalLocalFiles + totalDeletedFiles):N0}"); CommandLine.WriteLine($"Local Files: {totalLocalFiles:N0}"); CommandLine.WriteLine($"Deleted Files: {totalDeletedFiles:N0}"); CommandLine.WriteLine($"Total Objects: {totalObjects:N0}"); CommandLine.WriteLine($"Local Size: {Formatter.FromBytes(localSize)}"); - if (vault.FileSystem.Service.Equals(FileService.Local)) + if (vault.Credentials.Service.Equals(FileService.Local)) { - DriveInfo drive = new(vault.FileSystem.RootDirectory); + DriveInfo drive = new(vault.Credentials.RootDirectory); long diskUsage = drive.TotalSize - drive.TotalFreeSpace; CommandLine.WriteLine($"Total Usage: {Formatter.FromBytes(diskUsage)} ({Math.Round(diskUsage / (double)drive.TotalSize * 100, 1)}%)"); CommandLine.WriteLine($"Disk Free: {Formatter.FromBytes(drive.TotalFreeSpace)} ({Math.Round(drive.TotalFreeSpace / (double)drive.TotalSize * 100, 1)}%)"); diff --git a/Parallel.Cli/Commands/PullCommand.cs b/Parallel.Cli/Commands/PullCommand.cs index 6f6c1c9..314f63f 100644 --- a/Parallel.Cli/Commands/PullCommand.cs +++ b/Parallel.Cli/Commands/PullCommand.cs @@ -71,7 +71,7 @@ private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force Log.Debug($"Pulling {pullFiles.Count} files..."); await syncManager.PullFilesAsync(pullFiles.ToArray(), new ProgressReport(vault, files.Count())); - CommandLine.WriteLine(vault, $"Successfully pulled {pullFiles.Count:N0} files from '{vault.FileSystem.RootDirectory}'.", ConsoleColor.Green); + CommandLine.WriteLine(vault, $"Successfully pulled {pullFiles.Count:N0} files from '{vault.Credentials.RootDirectory}'.", ConsoleColor.Green); } private async Task PullFileAsync(ISyncManager syncManager, string fullPath, bool force) @@ -95,7 +95,7 @@ private async Task PullFileAsync(ISyncManager syncManager, string fullPath, bool await syncManager.PullFilesAsync([remoteFile], new ProgressLogger()); await syncManager.DisconnectAsync(); - CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pulled file from '{syncManager.RemoteVault.FileSystem.RootDirectory}'.", ConsoleColor.Green); + CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pulled file from '{syncManager.RemoteVault.Credentials.RootDirectory}'.", ConsoleColor.Green); } } } \ No newline at end of file diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index de0d753..e4288cd 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -39,7 +39,7 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") addCmd.SetHandler(() => { CommandLine.WriteLine("Creating new storage vault...", ConsoleColor.DarkGray); - FileSystemCredentials fsc = new FileSystemCredentials(); + StorageCredentials fsc = new StorageCredentials(); fsc.Service = Enum.Parse(CommandLine.ReadString($"Service ({string.Join(", ", Enum.GetNames(typeof(FileService)))})"), true); if (fsc.Service == FileService.Local) { diff --git a/Parallel.Core/Diagnostics/IProgressReporter.cs b/Parallel.Core/Diagnostics/IProgressReporter.cs index be7934f..ec41121 100644 --- a/Parallel.Core/Diagnostics/IProgressReporter.cs +++ b/Parallel.Core/Diagnostics/IProgressReporter.cs @@ -6,12 +6,10 @@ namespace Parallel.Core.Diagnostics { public enum ProgressOperation { - Archiving, - Pulling, - Pushing, - Compressing, - Decompressing, - Syncing + Archived, + Pulled, + Pushed, + Synced } /// diff --git a/Parallel.Core/IO/Blobs/BlobStorage.cs b/Parallel.Core/IO/Blobs/BlobStorage.cs index f34413f..b0c61af 100644 --- a/Parallel.Core/IO/Blobs/BlobStorage.cs +++ b/Parallel.Core/IO/Blobs/BlobStorage.cs @@ -3,6 +3,7 @@ using Parallel.Core.Diagnostics; using Parallel.Core.IO.FileSystem; using Parallel.Core.Security; +using Parallel.Core.Storage; namespace Parallel.Core.IO.Blobs { diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index fff8c25..4a49d74 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -6,6 +6,7 @@ using Parallel.Core.IO.FileSystem; using Parallel.Core.Models; using Parallel.Core.Settings; +using Parallel.Core.Storage; using Parallel.Core.Utils; namespace Parallel.Core.IO @@ -98,7 +99,7 @@ public static string Combine(params string[] paths) /// public static string GetRootDirectory(LocalVaultConfig localVault) { - return Combine(localVault.FileSystem.RootDirectory, "Parallel", localVault.Id); + return Combine(localVault.Credentials.RootDirectory, "Parallel", localVault.Id); } /// @@ -149,8 +150,8 @@ public static string GetDatabaseFile(LocalVaultConfig localVault) /// public static string Remote(string path, RemoteVaultConfig remoteVaultConfig) { - string root = Path.Combine(remoteVaultConfig.FileSystem.RootDirectory, "Parallel", remoteVaultConfig.Id, "Files", path.Replace(":", string.Empty)) + ".gz"; - return remoteVaultConfig.FileSystem.Service switch + string root = Path.Combine(remoteVaultConfig.Credentials.RootDirectory, "Parallel", remoteVaultConfig.Id, "Files", path.Replace(":", string.Empty)) + ".gz"; + return remoteVaultConfig.Credentials.Service switch { FileService.Local => root, FileService.Remote => root.Replace('\\', '/'), diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index f368ab2..561935b 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -5,6 +5,7 @@ using Parallel.Core.IO.FileSystem; using Parallel.Core.Models; using Parallel.Core.Settings; +using Parallel.Core.Storage; namespace Parallel.Core.IO.Syncing { @@ -27,7 +28,7 @@ public abstract class BaseSyncManager : ISyncManager public IDatabase Database { get; set; } /// - public IStorageProvider Storage { get; set; } + public IStorageProvider StorageProvider { get; set; } /// /// @@ -35,7 +36,7 @@ public abstract class BaseSyncManager : ISyncManager /// public BaseSyncManager(LocalVaultConfig localVault) { - Storage = StorageProvider.CreateNew(localVault); + StorageProvider = StorageConnection.CreateNew(localVault); LocalVault = localVault; } @@ -43,13 +44,13 @@ public BaseSyncManager(LocalVaultConfig localVault) public async Task ConnectAsync() { string root = PathBuilder.GetRootDirectory(LocalVault); - if (!await Storage.ExistsAsync(root)) + if (!await StorageProvider.ExistsAsync(root)) { - await Storage.CreateDirectoryAsync(root); + await StorageProvider.CreateDirectoryAsync(root); Log.Debug($"Created root directory: {root}"); } - if (!await Storage.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) + if (!await StorageProvider.ExistsAsync(PathBuilder.GetConfigurationFile(LocalVault))) { RemoteVault = new RemoteVaultConfig(LocalVault); RemoteVault.IgnoreDirectories.Add(PathBuilder.GetRootDirectory(LocalVault)); @@ -59,7 +60,7 @@ public async Task ConnectAsync() } else { - await Storage.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new NullProgressReporter()); + await StorageProvider.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new NullProgressReporter()); RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); if(config == null) return false; RemoteVault = config; @@ -67,7 +68,7 @@ public async Task ConnectAsync() Log.Debug($"Downloaded config file: {TempConfigFile}"); } - if (!await Storage.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) + if (!await StorageProvider.ExistsAsync(PathBuilder.GetDatabaseFile(LocalVault))) { Database = new SqliteContext(TempDbFile); await Database.InitializeAsync(); @@ -76,7 +77,7 @@ public async Task ConnectAsync() } else { - await Storage.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new NullProgressReporter()); + await StorageProvider.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new NullProgressReporter()); Database = new SqliteContext(TempDbFile); Log.Debug($"Downloaded db file: {TempDbFile}"); @@ -92,8 +93,8 @@ public async Task DisconnectAsync() Log.Debug($"Uploaded db file: {TempDbFile}"); SystemFile[] tempFiles = [new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault)), new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))]; - await Storage.UploadFilesAsync(tempFiles, new NullProgressReporter()); - Storage.Dispose(); + await StorageProvider.UploadFilesAsync(tempFiles, new NullProgressReporter()); + StorageProvider.Dispose(); } /// diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 7cfc03f..5a9ae31 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -24,27 +24,28 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter if (!files.Any()) return; SystemFile[] backupFiles = files.Where(f => !f.Deleted).ToArray(); Log.Information($"Backing up {backupFiles.Length} files..."); - await Storage.UploadFilesAsync(backupFiles, progress); + await StorageProvider.UploadFilesAsync(backupFiles, progress); progress.Reset(); await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { if (file.Deleted) { - progress.Report(ProgressOperation.Archiving, file); await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); await Database.AddFileAsync(file); + progress.Report(ProgressOperation.Archived, file); } else { - progress.Report(ProgressOperation.Syncing, file); - SystemFile? remote = await Storage.GetFileAsync(file.RemotePath); + SystemFile? remote = await StorageProvider.GetFileAsync(file.RemotePath); if (remote is not null) { file.RemoteSize = remote.RemoteSize; await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pushed); await Database.AddFileAsync(file); } + + progress.Report(ProgressOperation.Synced, file); } }); } @@ -52,7 +53,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options /// public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter progress) { - await Storage.DownloadFilesAsync(files, progress); + await StorageProvider.DownloadFilesAsync(files, progress); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index 80f93f6..315bfff 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -5,6 +5,7 @@ using Parallel.Core.IO.FileSystem; using Parallel.Core.Models; using Parallel.Core.Settings; +using Parallel.Core.Storage; namespace Parallel.Core.IO.Syncing { @@ -31,7 +32,7 @@ public interface ISyncManager /// /// The associated file system connection. /// - public IStorageProvider Storage { get; set; } + public IStorageProvider StorageProvider { get; set; } /// /// Establishes a connection to the associated and downloads the needed files. diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index a3e5f53..e3c9b32 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -13,6 +13,7 @@ using Parallel.Core.Security; using Parallel.Core.Settings; using Parallel.Core.Utils; +using Parallel.Core.Workers; namespace Parallel.Core.IO.Syncing { @@ -41,7 +42,7 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter TimeSpan uploadTimeout = TimeSpan.FromSeconds(30); TimeSpan writeBlockWarn = TimeSpan.FromSeconds(5); - Channel channel = Channel.CreateBounded(new BoundedChannelOptions(256) + Channel channel = Channel.CreateBounded(new BoundedChannelOptions(256) { FullMode = BoundedChannelFullMode.Wait, SingleReader = false, @@ -50,7 +51,7 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter Task[] workerTasks = Enumerable.Range(0, ParallelConfig.MaxStaticTransfers).Select(workerId => Task.Run(async () => { - await foreach (TransferWorker job in channel.Reader.ReadAllAsync()) + await foreach (UploadWorker job in channel.Reader.ReadAllAsync()) { Interlocked.Decrement(ref queued); @@ -58,23 +59,23 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter { Interlocked.Increment(ref total); string fullPath = PathBuilder.Combine(job.RemotePath, job.Filename); - if (await Storage.ExistsAsync(fullPath)) + if (await StorageProvider.ExistsAsync(fullPath)) { Log.Debug($"[WORKER {workerId}] UPLOAD SKIPPED: {fullPath}"); continue; } - if (!await Storage.ExistsAsync(job.RemotePath)) await Storage.CreateDirectoryAsync(job.RemotePath); + if (!await StorageProvider.ExistsAsync(job.RemotePath)) await StorageProvider.CreateDirectoryAsync(job.RemotePath); await using MemoryStream ms = new MemoryStream(job.Data, false); using CancellationTokenSource cts = new CancellationTokenSource(uploadTimeout); - Task uploadTask = Storage.UploadStreamAsync(ms, fullPath); + Task uploadTask = StorageProvider.UploadStreamAsync(ms, fullPath); Task timeoutTask = await Task.WhenAny(uploadTask, Task.Delay(uploadTimeout, cts.Token)); if (timeoutTask != uploadTask) { Log.Error($"[WORKER {workerId}] UPLOAD TIMEOUT: {fullPath}"); - job.OnError?.Invoke(new TimeoutException($"Upload timed out after {uploadTimeout}")); + job.OnException?.Invoke(new TimeoutException($"Upload timed out after {uploadTimeout}")); continue; } @@ -85,7 +86,7 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter catch (Exception ex) { Log.Error($"[WORKER {workerId}] UPLOAD ERROR: {job.Filename} : {ex}"); - job.OnError?.Invoke(ex); + job.OnException?.Invoke(ex); } } })).ToArray(); @@ -100,15 +101,13 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options { if (file.Deleted) { - progress.Report(ProgressOperation.Archiving, file); await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); await Database.AddFileAsync(file); + progress.Report(ProgressOperation.Archived, file); return; } - progress.Report(ProgressOperation.Pushing, file); await Database.AddFileAsync(file); - await using FileStream fs = new FileStream(file.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: ChunkSize, useAsync: true); byte[] buffer = ArrayPool.Shared.Rent(ChunkSize); @@ -123,13 +122,13 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string hash = HashGenerator.CreateSHA256(chunk); await Database.AddObjectAsync(file.Id, hash, index++); - string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string basePath = PathBuilder.Combine(RemoteVault.Credentials.RootDirectory, "Parallel", RemoteVault.Id, "objects"); string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - TransferWorker job = new(chunk, parentDir, hash[4..], ex => progress.Failed(ex, file)); + UploadWorker worker = new(chunk, parentDir, hash[4..], ex => progress.Failed(ex, file)); - Task writeTask = channel.Writer.WriteAsync(job, ct).AsTask(); + Task writeTask = channel.Writer.WriteAsync(worker, ct).AsTask(); Task timeoutTask = await Task.WhenAny(writeTask, Task.Delay(writeBlockWarn, ct)); if (timeoutTask != writeTask) { @@ -138,6 +137,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } Interlocked.Increment(ref queued); + progress.Report(ProgressOperation.Pushed, file); } } finally @@ -179,34 +179,32 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options /// public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter progress) { - await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + long queued = 0, completed = 0, total = 0; + TimeSpan downloadTimeout = TimeSpan.FromSeconds(30); + TimeSpan writeBlockWarn = TimeSpan.FromSeconds(5); + + Channel channel = Channel.CreateBounded(new BoundedChannelOptions(256) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = false, + SingleWriter = false + }); + + /*Task[] workerTasks = Enumerable.Range(0, ParallelConfig.MaxStaticTransfers).Select(workerId -> Task.Run(async () => { try { - progress.Report(ProgressOperation.Pulling, file); - string? parentDir = Path.GetDirectoryName(file.LocalPath); - if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); - - await using FileStream fs = File.Create(file.LocalPath); - foreach (string hash in await Database.GetObjectsAsync(file.Id)) + await foreach (DownloadWorker job in channel.Reader.ReadAllAsync()) { - string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); - string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); - if (await Storage.ExistsAsync(remotePath)) + Interlocked.Decrement(ref queued); + + try { - await Storage.DownloadStreamAsync(fs, remotePath); + Interlocked.Increment(ref total); } } - - await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pulled); - await fs.FlushAsync(ct); - } - catch (Exception ex) - { - Log.Error(ex.GetBaseException().ToString()); - progress.Failed(ex, file); } - }); + }));*/ } } } \ No newline at end of file diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs index 82d8203..d353242 100644 --- a/Parallel.Core/Settings/LocalVaultConfig.cs +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -2,6 +2,7 @@ using Parallel.Core.IO.FileSystem; using Parallel.Core.Security; +using Parallel.Core.Storage; namespace Parallel.Core.Settings { @@ -28,27 +29,27 @@ public class LocalVaultConfig /// /// The credentials needed to log in to the associated . /// - public FileSystemCredentials FileSystem { get; } + public StorageCredentials Credentials { get; } /// /// Initializes a new instance of the class. /// /// /// - /// + /// [JsonConstructor] - public LocalVaultConfig(string id, string name, FileSystemCredentials fileSystem) + public LocalVaultConfig(string id, string name, StorageCredentials credentials) { Id = id; Name = name; - FileSystem = fileSystem; + Credentials = credentials; } - public LocalVaultConfig(string name, FileSystemCredentials fileSystem) + public LocalVaultConfig(string name, StorageCredentials credentials) { Id = HashGenerator.GenerateHash(8, true); Name = name; - FileSystem = fileSystem; + Credentials = credentials; } /// diff --git a/Parallel.Core/Settings/RemoteVaultConfig.cs b/Parallel.Core/Settings/RemoteVaultConfig.cs index d31cba8..8be5dac 100644 --- a/Parallel.Core/Settings/RemoteVaultConfig.cs +++ b/Parallel.Core/Settings/RemoteVaultConfig.cs @@ -48,12 +48,12 @@ public class RemoteVaultConfig : LocalVaultConfig public HashSet PruneDirectories { get; } = []; - public RemoteVaultConfig(LocalVaultConfig localVault) : base(localVault.Id, localVault.Name, localVault.FileSystem) { } + public RemoteVaultConfig(LocalVaultConfig localVault) : base(localVault.Id, localVault.Name, localVault.Credentials) { } - public RemoteVaultConfig(string profileName, FileSystemCredentials fsc) : base(profileName, fsc) { } + public RemoteVaultConfig(string profileName, StorageCredentials fsc) : base(profileName, fsc) { } [JsonConstructor] - public RemoteVaultConfig(string id, string name, FileSystemCredentials fileSystem, int backupInterval, int prunePeriod, IEnumerable backupDirectories, IEnumerable ignoreDirectories, IEnumerable pruneDirectories) : base(id, name, fileSystem) + public RemoteVaultConfig(string id, string name, StorageCredentials credentials, int backupInterval, int prunePeriod, IEnumerable backupDirectories, IEnumerable ignoreDirectories, IEnumerable pruneDirectories) : base(id, name, credentials) { BackupInterval = backupInterval; PrunePeriod = prunePeriod; diff --git a/Parallel.Core/Settings/FileSystemCredentials.cs b/Parallel.Core/Settings/StorageCredentials.cs similarity index 85% rename from Parallel.Core/Settings/FileSystemCredentials.cs rename to Parallel.Core/Settings/StorageCredentials.cs index ac7d4f9..4b0209c 100644 --- a/Parallel.Core/Settings/FileSystemCredentials.cs +++ b/Parallel.Core/Settings/StorageCredentials.cs @@ -3,13 +3,14 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Parallel.Core.IO.FileSystem; +using Parallel.Core.Storage; namespace Parallel.Core.Settings { /// /// Represents credentials used to gain access to various s. /// - public class FileSystemCredentials + public class StorageCredentials { public FileService Service { get; set; } = FileService.Local; public string RootDirectory { get; set; } = string.Empty; @@ -27,9 +28,9 @@ public class FileSystemCredentials /// public string? EncryptionKey { get; set; } = null; - public FileSystemCredentials() { } + public StorageCredentials() { } - public FileSystemCredentials(string root) + public StorageCredentials(string root) { RootDirectory = root; } diff --git a/Parallel.Core/Storage/IStorageProvider.cs b/Parallel.Core/Storage/IStorageProvider.cs index 7c6ce3f..345f94a 100644 --- a/Parallel.Core/Storage/IStorageProvider.cs +++ b/Parallel.Core/Storage/IStorageProvider.cs @@ -1,15 +1,9 @@ // Copyright 2025 Kyle Ebbinga -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Parallel.Core.Diagnostics; -using Parallel.Core.IO.Blobs; using Parallel.Core.Models; -namespace Parallel.Core.IO.FileSystem +namespace Parallel.Core.Storage { /// /// Defines the way for communicating with a file system. diff --git a/Parallel.Core/Storage/LocalStorageProvider.cs b/Parallel.Core/Storage/LocalStorageProvider.cs index 3fbfe18..7352463 100644 --- a/Parallel.Core/Storage/LocalStorageProvider.cs +++ b/Parallel.Core/Storage/LocalStorageProvider.cs @@ -1,17 +1,11 @@ // Copyright 2025 Kyle Ebbinga -using System.Diagnostics; using System.IO.Compression; -using Microsoft.VisualBasic.FileIO; -using Newtonsoft.Json.Linq; using Parallel.Core.Diagnostics; -using Parallel.Core.IO.Blobs; using Parallel.Core.Models; using Parallel.Core.Settings; -using Parallel.Core.Utils; -using SearchOption = System.IO.SearchOption; -namespace Parallel.Core.IO.FileSystem +namespace Parallel.Core.Storage { /// /// Represents the wrapper for a default dotnet file system. diff --git a/Parallel.Core/Storage/SshStorageProvider.cs b/Parallel.Core/Storage/SshStorageProvider.cs index 28847c2..65e39a3 100644 --- a/Parallel.Core/Storage/SshStorageProvider.cs +++ b/Parallel.Core/Storage/SshStorageProvider.cs @@ -1,18 +1,15 @@ // Copyright 2025 Kyle Ebbinga using System.Diagnostics; -using Parallel.Core.Settings; -using Renci.SshNet; -using Renci.SshNet.Sftp; using System.IO.Compression; -using Newtonsoft.Json.Linq; using Parallel.Core.Diagnostics; -using Parallel.Core.IO.Blobs; using Parallel.Core.Models; using Parallel.Core.Security; -using Parallel.Core.Utils; +using Parallel.Core.Settings; +using Renci.SshNet; +using Renci.SshNet.Sftp; -namespace Parallel.Core.IO.FileSystem +namespace Parallel.Core.Storage { /// /// Represents the wrapper for an SFTP file system through SSH. @@ -28,7 +25,7 @@ public class SshStorageProvider : IStorageProvider /// The credentials to log in with. public SshStorageProvider(LocalVaultConfig localVault) { - _connectionInfo = new ConnectionInfo(localVault.FileSystem.Address, localVault.FileSystem.Username, new PasswordAuthenticationMethod(localVault.FileSystem.Username, Encryption.Decode(localVault.FileSystem.Password))); + _connectionInfo = new ConnectionInfo(localVault.Credentials.Address, localVault.Credentials.Username, new PasswordAuthenticationMethod(localVault.Credentials.Username, Encryption.Decode(localVault.Credentials.Password))); _client = new SftpClient(_connectionInfo); _client.Connect(); } diff --git a/Parallel.Core/Storage/StorageProvider.cs b/Parallel.Core/Storage/StorageConnection.cs similarity index 91% rename from Parallel.Core/Storage/StorageProvider.cs rename to Parallel.Core/Storage/StorageConnection.cs index 9ad1f64..c73e75e 100644 --- a/Parallel.Core/Storage/StorageProvider.cs +++ b/Parallel.Core/Storage/StorageConnection.cs @@ -1,6 +1,7 @@ // Copyright 2025 Kyle Ebbinga using Parallel.Core.Settings; +using Parallel.Core.Storage; namespace Parallel.Core.IO.FileSystem { @@ -28,7 +29,7 @@ public enum FileService /// /// Represents the way to connect to different file system associations. This class cannot be inherited. /// - public static class StorageProvider + public static class StorageConnection { /// /// Creates a new file system association. @@ -36,7 +37,7 @@ public static class StorageProvider /// The vault needed for the associated file system. public static IStorageProvider CreateNew(LocalVaultConfig vaultConfig) { - return vaultConfig.FileSystem.Service switch + return vaultConfig.Credentials.Service switch { FileService.Local => new LocalStorageProvider(vaultConfig), FileService.Remote => new SshStorageProvider(vaultConfig), diff --git a/Parallel.Core/Utils/TransferWorker.cs b/Parallel.Core/Utils/TransferWorker.cs deleted file mode 100644 index 301f5c1..0000000 --- a/Parallel.Core/Utils/TransferWorker.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -namespace Parallel.Core.Utils -{ - public class TransferWorker - { - public byte[] Data { get; } - public string RemotePath { get; } - public string Filename { get; } - public Action? OnError { get; } - - public TransferWorker(byte[] data, string remotePath, string filename, Action? onError = null) - { - Data = data; - RemotePath = remotePath; - Filename = filename; - OnError = onError; - } - } -} \ No newline at end of file diff --git a/Parallel.Core/Workers/BaseWorker.cs b/Parallel.Core/Workers/BaseWorker.cs new file mode 100644 index 0000000..054833b --- /dev/null +++ b/Parallel.Core/Workers/BaseWorker.cs @@ -0,0 +1,33 @@ +// Copyright 2025 Kyle Ebbinga + +using Parallel.Core.Storage; + +namespace Parallel.Core.Workers +{ + /// + /// Represents + /// + public abstract class BaseWorker + { + /// + /// Gets the remote path in the . + /// + public string RemotePath { get; } + + /// + /// The to run when an is thrown. + /// + public Action? OnException { get; } + + /// + /// Initializes a new instance of the class with a remote path. + /// + /// + /// + protected BaseWorker(string remotePath, Action? onException) + { + RemotePath = remotePath; + OnException = onException; + } + } +} \ No newline at end of file diff --git a/Parallel.Core/Workers/DownloadWorker.cs b/Parallel.Core/Workers/DownloadWorker.cs new file mode 100644 index 0000000..b3fb2c8 --- /dev/null +++ b/Parallel.Core/Workers/DownloadWorker.cs @@ -0,0 +1,14 @@ +// Copyright 2025 Kyle Ebbinga + +namespace Parallel.Core.Workers +{ + public class DownloadWorker : BaseWorker + { + public FileAggregator Aggregator { get; } + + public DownloadWorker(string remotePath, Action? onException = null) : base(remotePath, onException) + { + + } + } +} \ No newline at end of file diff --git a/Parallel.Core/Workers/FileAggregator.cs b/Parallel.Core/Workers/FileAggregator.cs new file mode 100644 index 0000000..3410274 --- /dev/null +++ b/Parallel.Core/Workers/FileAggregator.cs @@ -0,0 +1,13 @@ +// Copyright 2025 Kyle Ebbinga + +namespace Parallel.Core.Workers +{ + public class FileAggregator + { + private readonly string _localPath; + private readonly byte[][] _chunks; + private readonly Action? _onComplete; + private readonly Action? _onError; + private int _count; + } +} \ No newline at end of file diff --git a/Parallel.Core/Workers/UploadWorker.cs b/Parallel.Core/Workers/UploadWorker.cs new file mode 100644 index 0000000..6ee3918 --- /dev/null +++ b/Parallel.Core/Workers/UploadWorker.cs @@ -0,0 +1,22 @@ +// Copyright 2025 Kyle Ebbinga + +namespace Parallel.Core.Workers +{ + /// + /// Represents a worker responsible for uploading files. + /// + public class UploadWorker : BaseWorker + { + public byte[] Data { get; } + + public string Filename { get; } + + + public UploadWorker(byte[] data, string filename, string remotePath, Action? onError = null) : base(remotePath, onError) + { + Data = data; + Filename = filename; + + } + } +} \ No newline at end of file From 3fd87172067172a4f740f756e3d0ecdc26f4f652 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 04:16:06 -0600 Subject: [PATCH 10/17] Fixed `System.NullReferenceException` when no config has been created. --- Parallel.Cli/Commands/DiskCommand.cs | 4 ++-- Parallel.Cli/Commands/PullCommand.cs | 4 ++-- Parallel.Cli/Commands/PushCommand.cs | 4 ++-- Parallel.Cli/Commands/RemapCommand.cs | 4 ++-- Parallel.Cli/Commands/VaultsCommand.cs | 12 ++++++------ Parallel.Core/IO/Syncing/SyncManager.cs | 3 ++- Parallel.Core/Storage/StorageConnection.cs | 4 ++-- 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Parallel.Cli/Commands/DiskCommand.cs b/Parallel.Cli/Commands/DiskCommand.cs index b2372f8..0a62b6d 100644 --- a/Parallel.Cli/Commands/DiskCommand.cs +++ b/Parallel.Cli/Commands/DiskCommand.cs @@ -36,8 +36,8 @@ public DiskCommand() : base("disk", "Shows the current disk usage.") private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) { CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); - ISyncManager syncManager = SyncManager.CreateNew(vault); - if (!await syncManager.ConnectAsync()) + ISyncManager? syncManager = SyncManager.CreateNew(vault); + if (syncManager == null || !await syncManager.ConnectAsync()) { CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); return; diff --git a/Parallel.Cli/Commands/PullCommand.cs b/Parallel.Cli/Commands/PullCommand.cs index 314f63f..17a9ad2 100644 --- a/Parallel.Cli/Commands/PullCommand.cs +++ b/Parallel.Cli/Commands/PullCommand.cs @@ -40,8 +40,8 @@ public PullCommand() : base("pull", "Pulls changes from a vault.") private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force) { CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); - ISyncManager syncManager = SyncManager.CreateNew(vault); - if (!await syncManager.ConnectAsync()) + ISyncManager? syncManager = SyncManager.CreateNew(vault); + if (syncManager == null || !await syncManager.ConnectAsync()) { CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); return; diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 2f125de..33e7b4d 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -54,8 +54,8 @@ private async Task SyncPathAsync(string path) CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); await Program.Settings.ForEachVaultAsync(async vault => { - ISyncManager syncManager = SyncManager.CreateNew(vault); - if (!await syncManager.ConnectAsync()) + ISyncManager? syncManager = SyncManager.CreateNew(vault); + if (syncManager == null || !await syncManager.ConnectAsync()) { CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); return; diff --git a/Parallel.Cli/Commands/RemapCommand.cs b/Parallel.Cli/Commands/RemapCommand.cs index 8641ae5..8d0c68e 100644 --- a/Parallel.Cli/Commands/RemapCommand.cs +++ b/Parallel.Cli/Commands/RemapCommand.cs @@ -48,8 +48,8 @@ public RemapCommand() : base("remap", "Remaps paths in the vault.") private async Task RemapPathAsync(LocalVaultConfig vault, string source, string target) { - ISyncManager syncManager = SyncManager.CreateNew(vault); - if (!await syncManager.ConnectAsync()) + ISyncManager? syncManager = SyncManager.CreateNew(vault); + if (syncManager == null || !await syncManager.ConnectAsync()) { CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); return; diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index e4288cd..25386fd 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -78,20 +78,20 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") this.AddCommand(viewCmd); viewCmd.AddArgument(configArg); - viewCmd.SetHandler(async (vault) => + viewCmd.SetHandler(async (config) => { CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); - LocalVaultConfig? config = ParallelConfig.GetVault(vault); - if (config == null) + LocalVaultConfig? vault = ParallelConfig.GetVault(config); + if (vault == null) { CommandLine.WriteLine($"Unable to find vault with name: '{vault}'", ConsoleColor.Yellow); return; } - ISyncManager syncManager = SyncManager.CreateNew(config); - if (!await syncManager.ConnectAsync()) + ISyncManager? syncManager = SyncManager.CreateNew(vault); + if (syncManager == null || !await syncManager.ConnectAsync()) { - CommandLine.WriteLine(config, $"Failed to connect to vault '{config.Name}'!", ConsoleColor.Red); + CommandLine.WriteLine(vault, $"Failed to connect to vault '{vault.Name}'!", ConsoleColor.Red); return; } diff --git a/Parallel.Core/IO/Syncing/SyncManager.cs b/Parallel.Core/IO/Syncing/SyncManager.cs index 842fe2d..90bd711 100644 --- a/Parallel.Core/IO/Syncing/SyncManager.cs +++ b/Parallel.Core/IO/Syncing/SyncManager.cs @@ -14,8 +14,9 @@ public static class SyncManager /// /// /// - public static ISyncManager CreateNew(LocalVaultConfig localVault) + public static ISyncManager? CreateNew(LocalVaultConfig? localVault) { + if (localVault?.Credentials is null) return null; return new ObjectSyncManager(localVault); } } diff --git a/Parallel.Core/Storage/StorageConnection.cs b/Parallel.Core/Storage/StorageConnection.cs index c73e75e..6798286 100644 --- a/Parallel.Core/Storage/StorageConnection.cs +++ b/Parallel.Core/Storage/StorageConnection.cs @@ -37,12 +37,12 @@ public static class StorageConnection /// The vault needed for the associated file system. public static IStorageProvider CreateNew(LocalVaultConfig vaultConfig) { - return vaultConfig.Credentials.Service switch + return vaultConfig?.Credentials.Service switch { FileService.Local => new LocalStorageProvider(vaultConfig), FileService.Remote => new SshStorageProvider(vaultConfig), //FileService.Cloud => new AmazonS3FileSystem(credentials), - _ => null + _ => throw new ArgumentOutOfRangeException() }; } } From abc045abfe396d19bc4d30ae9c65cbc708a77ded Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 04:16:48 -0600 Subject: [PATCH 11/17] Updated version to 0.0.0.0. Now is set by GitHub actions. --- Parallel.Cli/Parallel.Cli.csproj | 2 +- Parallel.Core/Parallel.Core.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Parallel.Cli/Parallel.Cli.csproj b/Parallel.Cli/Parallel.Cli.csproj index 7c0c6b5..892cf76 100644 --- a/Parallel.Cli/Parallel.Cli.csproj +++ b/Parallel.Cli/Parallel.Cli.csproj @@ -8,7 +8,7 @@ enable enable Parallel - 1.0.0 + 0.0.0.0 Kyle Ebbinga Copyright $(Company). All Rights Reserved. $(AssemblyVersion) diff --git a/Parallel.Core/Parallel.Core.csproj b/Parallel.Core/Parallel.Core.csproj index fdd0ec4..21e6e5a 100644 --- a/Parallel.Core/Parallel.Core.csproj +++ b/Parallel.Core/Parallel.Core.csproj @@ -8,7 +8,7 @@ AnyCPU enable Parallel.Core - 1.0.0.0 + 0.0.0.0 Kyle Ebbinga Copyright $(Company). All Rights Reserved. $(AssemblyVersion) From 21a5a409bdd68f3392d6b05c401d07898b1fc982 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 05:20:13 -0600 Subject: [PATCH 12/17] Reformatted code with ReSharper --- Parallel.Cli/Commands/DuplicatesCommand.cs | 2 +- Parallel.Cli/Commands/VaultsCommand.cs | 48 +++++++------ Parallel.Cli/Parallel.Cli.csproj | 14 ++-- Parallel.Cli/Program.cs | 6 +- Parallel.Cli/Utils/CommandLine.cs | 4 +- Parallel.Cli/Utils/ProgressReport.cs | 2 +- .../Database/Contexts/SqliteContext.cs | 5 +- Parallel.Core/Database/IDatabase.cs | 1 - Parallel.Core/IO/Blobs/BlobStorage.cs | 72 ------------------- Parallel.Core/IO/Blobs/FileManifest.cs | 22 ------ Parallel.Core/IO/PathBuilder.cs | 2 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 2 +- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 1 - Parallel.Core/IO/Syncing/SyncManager.cs | 3 +- Parallel.Core/Parallel.Core.csproj | 16 ++--- Parallel.Core/Settings/LocalVaultConfig.cs | 7 -- Parallel.Core/Settings/RemoteVaultConfig.cs | 2 - Parallel.Core/Storage/LocalStorageProvider.cs | 3 +- Parallel.Core/Storage/SshStorageProvider.cs | 2 +- 19 files changed, 57 insertions(+), 157 deletions(-) delete mode 100644 Parallel.Core/IO/Blobs/BlobStorage.cs delete mode 100644 Parallel.Core/IO/Blobs/FileManifest.cs diff --git a/Parallel.Cli/Commands/DuplicatesCommand.cs b/Parallel.Cli/Commands/DuplicatesCommand.cs index d5f1393..05136b9 100644 --- a/Parallel.Cli/Commands/DuplicatesCommand.cs +++ b/Parallel.Cli/Commands/DuplicatesCommand.cs @@ -12,7 +12,7 @@ namespace Parallel.Cli.Commands { public class DuplicatesCommand : Command { - private Argument sourceArg = new("path", "The directory to scan."); + private readonly Argument sourceArg = new("path", "The directory to scan."); public DuplicatesCommand() : base("duplicates", "Scans a directory for duplicate files.") { diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index 25386fd..6851a69 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -16,11 +16,11 @@ public class VaultsCommand : Command private readonly Argument configArg = new("config", "The vault configuration to use."); private readonly Option configOpt = new(["--config", "-c"], "The vault configuration to use."); - private Command addCmd = new("add", "Adds a new vault configuration."); + private readonly Command addCmd = new("add", "Adds a new vault configuration."); private Command editCmd = new("edit", "Edits a vault configuration."); private readonly Command findCmd = new("find", "Finds vault configurations in a location."); - private Command viewCmd = new("view", "Shows the vault configuration."); - private Command setCmd = new("set", "Sets a new vault configuration."); + private readonly Command viewCmd = new("view", "Shows the vault configuration."); + private readonly Command setCmd = new("set", "Sets a new vault configuration."); private Command delCmd = new("delete", "Deletes a vault configuration."); public VaultsCommand() : base("vaults", "View or edit the vaults.") @@ -39,31 +39,35 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") addCmd.SetHandler(() => { CommandLine.WriteLine("Creating new storage vault...", ConsoleColor.DarkGray); - StorageCredentials fsc = new StorageCredentials(); - fsc.Service = Enum.Parse(CommandLine.ReadString($"Service ({string.Join(", ", Enum.GetNames(typeof(FileService)))})"), true); - if (fsc.Service == FileService.Local) + StorageCredentials spc = new StorageCredentials { - fsc.RootDirectory = CommandLine.ReadString("Root"); + Service = Enum.Parse(CommandLine.ReadString($"Service ({string.Join(", ", Enum.GetNames(typeof(FileService)))})") ?? string.Empty, true) + }; + + string profileId = CommandLine.ReadString("Id") ?? HashGenerator.GenerateHash(8, true); + string profileName = CommandLine.ReadString("Name") ?? "Default"; + if (spc.Service == FileService.Local) + { + spc.RootDirectory = CommandLine.ReadString("Root") ?? string.Empty; } - else if (fsc.Service == FileService.Cloud) + else if (spc.Service == FileService.Cloud) { - fsc.Address = CommandLine.ReadString("Bucket Name"); - fsc.Username = CommandLine.ReadString("Access Key"); - fsc.Password = CommandLine.ReadPassword("Secret Key"); + spc.Address = CommandLine.ReadString("Bucket Name"); + spc.Username = CommandLine.ReadString("Access Key"); + spc.Password = CommandLine.ReadPassword("Secret Key"); } else { - fsc.RootDirectory = CommandLine.ReadString("Root"); - fsc.Address = CommandLine.ReadString("Address"); - fsc.Username = CommandLine.ReadString("Username"); - fsc.Password = CommandLine.ReadPassword("Password"); + spc.RootDirectory = CommandLine.ReadString("Root") ?? string.Empty; + spc.Address = CommandLine.ReadString("Address"); + spc.Username = CommandLine.ReadString("Username"); + spc.Password = CommandLine.ReadPassword("Password"); } - //fsc.Encrypt = CommandLine.ReadBool("Encrypt files? (y/n)", false); - //fsc.EncryptionKey = HashGenerator.GenerateHash(32, true); + //spc.Encrypt = CommandLine.ReadBool("Encrypt files? (y/n)", false); + //spc.EncryptionKey = HashGenerator.GenerateHash(32, true); - string? profileName = CommandLine.ReadString("Profile Name"); - LocalVaultConfig localVault = new LocalVaultConfig(profileName, fsc); + LocalVaultConfig localVault = new(profileId, profileName, spc); Program.Settings.Vaults.Add(localVault); Program.Settings.Save(); @@ -97,9 +101,9 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") RemoteVaultConfig remoteVault = syncManager.RemoteVault; CommandLine.WriteLine($"'{remoteVault.Name}' ({remoteVault.Id}):"); - CommandLine.WriteArray($"Backup Directories", remoteVault.BackupDirectories); - CommandLine.WriteArray($"Ignore Directories", remoteVault.IgnoreDirectories); - CommandLine.WriteArray($"Prune Directories", remoteVault.PruneDirectories); + CommandLine.WriteArray("Backup Directories", remoteVault.BackupDirectories); + CommandLine.WriteArray("Ignore Directories", remoteVault.IgnoreDirectories); + CommandLine.WriteArray("Prune Directories", remoteVault.PruneDirectories); CommandLine.WriteLine($"Prune Period: {remoteVault.PrunePeriod} days"); await syncManager.DisconnectAsync(); diff --git a/Parallel.Cli/Parallel.Cli.csproj b/Parallel.Cli/Parallel.Cli.csproj index 892cf76..b655695 100644 --- a/Parallel.Cli/Parallel.Cli.csproj +++ b/Parallel.Cli/Parallel.Cli.csproj @@ -35,19 +35,19 @@ - - - - + + + + - - + + - + diff --git a/Parallel.Cli/Program.cs b/Parallel.Cli/Program.cs index 6aa1da6..24b7e4a 100644 --- a/Parallel.Cli/Program.cs +++ b/Parallel.Cli/Program.cs @@ -16,11 +16,11 @@ public static async Task Main(string[] args) Settings = ParallelConfig.Load(); string logFile = Path.Combine(PathBuilder.ProgramData, "Logs", "latest.txt"); if (File.Exists(logFile)) File.Delete(logFile); - #if DEBUG +#if DEBUG Log.Logger = new LoggerConfiguration().MinimumLevel.Debug().WriteTo.Console().CreateLogger(); - #else +#else Log.Logger = new LoggerConfiguration().MinimumLevel.Warning().WriteTo.File(logFile).CreateLogger(); - #endif +#endif AssemblyName assembly = Assembly.GetExecutingAssembly().GetName(); Log.Information($"{assembly.Name} [Version {assembly.Version}]"); diff --git a/Parallel.Cli/Utils/CommandLine.cs b/Parallel.Cli/Utils/CommandLine.cs index b11f710..fed2b37 100644 --- a/Parallel.Cli/Utils/CommandLine.cs +++ b/Parallel.Cli/Utils/CommandLine.cs @@ -11,12 +11,12 @@ public class CommandLine { private static readonly object _consoleLock = new(); - public static string ReadString(object value, ConsoleColor color = ConsoleColor.Gray) + public static string? ReadString(object value, ConsoleColor color = ConsoleColor.Gray) { Console.ForegroundColor = color; Console.Write($"> {value}: "); Console.ResetColor(); - return Console.ReadLine() ?? string.Empty; + return Console.ReadLine(); } public static bool ReadBool(object value, bool defaultValue, ConsoleColor color = ConsoleColor.Gray) diff --git a/Parallel.Cli/Utils/ProgressReport.cs b/Parallel.Cli/Utils/ProgressReport.cs index 5fe0409..623c28e 100644 --- a/Parallel.Cli/Utils/ProgressReport.cs +++ b/Parallel.Cli/Utils/ProgressReport.cs @@ -10,7 +10,7 @@ public class ProgressReport : IProgressReporter { private readonly LocalVaultConfig _localVault; private int _current; - private int _total; + private readonly int _total; public ProgressReport(LocalVaultConfig localVault, int totalFiles) { diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 9f4e133..d6cbb05 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -6,7 +6,6 @@ using Dapper; using Newtonsoft.Json.Linq; using Parallel.Core.IO; -using Parallel.Core.IO.Blobs; using Parallel.Core.Models; using Parallel.Core.Settings; using Parallel.Core.Utils; @@ -43,7 +42,9 @@ public async Task InitializeAsync() using IDbConnection connection = CreateConnection(); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `objects` (`id` TEXT NOT NULL, `hash` TEXT NOT NULL, orderIndex INTEGER NOT NULL, UNIQUE (id, orderIndex));"); - await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` BLOB, PRIMARY KEY(`id`));"); + await connection.ExecuteAsync( + "CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` BLOB, PRIMARY KEY(`id`));"); + await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `history` (`timestamp` LONG INTEGER NOT NULL, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`timestamp`));"); } diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 2ef3b69..061cfb9 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -3,7 +3,6 @@ using Parallel.Core.IO; using System; using System.Data; -using Parallel.Core.IO.Blobs; using Parallel.Core.IO.FileSystem; using Parallel.Core.Models; diff --git a/Parallel.Core/IO/Blobs/BlobStorage.cs b/Parallel.Core/IO/Blobs/BlobStorage.cs deleted file mode 100644 index b0c61af..0000000 --- a/Parallel.Core/IO/Blobs/BlobStorage.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using Parallel.Core.Diagnostics; -using Parallel.Core.IO.FileSystem; -using Parallel.Core.Security; -using Parallel.Core.Storage; - -namespace Parallel.Core.IO.Blobs -{ - /// - /// Represents the way to chunk files into blobs for syncing. - /// - public abstract class BlobStorage - { - /// - /// The size, in bytes, to use for chunks of a file. - /// - private static readonly int ChunkSize = 4194304; - - /// - /// Chunks a file into hashes for blob storage. - /// - /// The source path of the file. - /// The temp directory to send chunked objects to. - /// - /// - public static async Task CreateManifestAsync(IStorageProvider fileSystem, string sourcePath, string tempObjDir, IProgressReporter progress) - { - List chunkHashes = new List(); - await using FileStream fs = File.OpenRead(sourcePath); - byte[] buffer = new byte[ChunkSize]; - int bytesRead = 0; - - while ((bytesRead = await fs.ReadAsync(buffer)) > 0) - { - byte[] chunkData = new byte[bytesRead]; - Buffer.BlockCopy(buffer, 0, chunkData, 0, bytesRead); - - string hash = HashGenerator.CreateSHA256(chunkData); - string chunkPath = PathBuilder.GetObjectPath(tempObjDir, hash); - - if(!File.Exists(chunkPath)) await File.WriteAllBytesAsync(chunkPath, chunkData); - chunkHashes.Add(hash); - } - - Log.Debug($"Wrote {chunkHashes.Count} hashes to {tempObjDir}"); - return new FileManifest(sourcePath, chunkHashes, new FileInfo(sourcePath).Length); - } - - /// - /// Assembles a file from the chunked hashes. - /// - /// - /// The path to the chunked objects' folder. - /// - public async Task AssembleFileAsync(IEnumerable chunkHashes, string sourcePath, string createFilePath) - { - Log.Debug($"Assembling '{createFilePath}' from {chunkHashes.Count()} hashes."); - await using FileStream createStream = File.Create(createFilePath); - foreach (string hash in chunkHashes) - { - string chunkPath = PathBuilder.GetObjectPath(sourcePath, hash); - if(!File.Exists(chunkPath)) throw new FileNotFoundException($"Missing chunk for hash: {hash}"); - - await using FileStream chunkStream = File.OpenRead(chunkPath); - await chunkStream.CopyToAsync(createStream); - } - - await createStream.FlushAsync(); - } - } -} \ No newline at end of file diff --git a/Parallel.Core/IO/Blobs/FileManifest.cs b/Parallel.Core/IO/Blobs/FileManifest.cs deleted file mode 100644 index d2afcec..0000000 --- a/Parallel.Core/IO/Blobs/FileManifest.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2025 Kyle Ebbinga - -using Parallel.Core.Security; - -namespace Parallel.Core.IO.Blobs -{ - public class FileManifest - { - public string Id { get; } - public string Fullname { get; set; } - public List Hashes { get; } - public long Length { get; set; } - - public FileManifest(string path, IEnumerable hashes, long length) - { - Id = HashGenerator.CreateSHA1(path); - Fullname = path; - Hashes = hashes.ToList(); - Length = length; - } - } -} \ No newline at end of file diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index 4a49d74..4c17568 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -172,7 +172,7 @@ public static bool IsFile(string path) public static string GetObjectPath(string basePath, string hash) { string parentDir = Path.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); - if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); + if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); return Path.Combine(parentDir, hash[4..]); } } diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index 561935b..95b7ae9 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -62,7 +62,7 @@ public async Task ConnectAsync() { await StorageProvider.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new NullProgressReporter()); RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); - if(config == null) return false; + if (config == null) return false; RemoteVault = config; Log.Debug($"Downloaded config file: {TempConfigFile}"); diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index e3c9b32..c5d9c0f 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -8,7 +8,6 @@ using Newtonsoft.Json.Linq; using Parallel.Core.Database; using Parallel.Core.Diagnostics; -using Parallel.Core.IO.Blobs; using Parallel.Core.Models; using Parallel.Core.Security; using Parallel.Core.Settings; diff --git a/Parallel.Core/IO/Syncing/SyncManager.cs b/Parallel.Core/IO/Syncing/SyncManager.cs index 90bd711..1d91dc4 100644 --- a/Parallel.Core/IO/Syncing/SyncManager.cs +++ b/Parallel.Core/IO/Syncing/SyncManager.cs @@ -16,8 +16,7 @@ public static class SyncManager /// public static ISyncManager? CreateNew(LocalVaultConfig? localVault) { - if (localVault?.Credentials is null) return null; - return new ObjectSyncManager(localVault); + return localVault?.Credentials is null ? null : new ObjectSyncManager(localVault); } } } \ No newline at end of file diff --git a/Parallel.Core/Parallel.Core.csproj b/Parallel.Core/Parallel.Core.csproj index 21e6e5a..81f4173 100644 --- a/Parallel.Core/Parallel.Core.csproj +++ b/Parallel.Core/Parallel.Core.csproj @@ -17,12 +17,12 @@ - - - - - - + + + + + + @@ -35,8 +35,8 @@ - - + + diff --git a/Parallel.Core/Settings/LocalVaultConfig.cs b/Parallel.Core/Settings/LocalVaultConfig.cs index d353242..8565a50 100644 --- a/Parallel.Core/Settings/LocalVaultConfig.cs +++ b/Parallel.Core/Settings/LocalVaultConfig.cs @@ -45,13 +45,6 @@ public LocalVaultConfig(string id, string name, StorageCredentials credentials) Credentials = credentials; } - public LocalVaultConfig(string name, StorageCredentials credentials) - { - Id = HashGenerator.GenerateHash(8, true); - Name = name; - Credentials = credentials; - } - /// /// Loads settings from a file. /// diff --git a/Parallel.Core/Settings/RemoteVaultConfig.cs b/Parallel.Core/Settings/RemoteVaultConfig.cs index 8be5dac..a47406f 100644 --- a/Parallel.Core/Settings/RemoteVaultConfig.cs +++ b/Parallel.Core/Settings/RemoteVaultConfig.cs @@ -50,8 +50,6 @@ public class RemoteVaultConfig : LocalVaultConfig public RemoteVaultConfig(LocalVaultConfig localVault) : base(localVault.Id, localVault.Name, localVault.Credentials) { } - public RemoteVaultConfig(string profileName, StorageCredentials fsc) : base(profileName, fsc) { } - [JsonConstructor] public RemoteVaultConfig(string id, string name, StorageCredentials credentials, int backupInterval, int prunePeriod, IEnumerable backupDirectories, IEnumerable ignoreDirectories, IEnumerable pruneDirectories) : base(id, name, credentials) { diff --git a/Parallel.Core/Storage/LocalStorageProvider.cs b/Parallel.Core/Storage/LocalStorageProvider.cs index 7352463..33a9f4e 100644 --- a/Parallel.Core/Storage/LocalStorageProvider.cs +++ b/Parallel.Core/Storage/LocalStorageProvider.cs @@ -78,7 +78,7 @@ public Task ExistsAsync(string path) /// public Task GetFileAsync(string path) { - if(!File.Exists(path)) return Task.FromResult(null); + if (!File.Exists(path)) return Task.FromResult(null); FileInfo fi = new(path); SystemFile file = new SystemFile(path) @@ -87,6 +87,7 @@ public Task ExistsAsync(string path) RemotePath = fi.FullName, RemoteSize = fi.Length }; + return Task.FromResult(file); } diff --git a/Parallel.Core/Storage/SshStorageProvider.cs b/Parallel.Core/Storage/SshStorageProvider.cs index 65e39a3..411e026 100644 --- a/Parallel.Core/Storage/SshStorageProvider.cs +++ b/Parallel.Core/Storage/SshStorageProvider.cs @@ -120,7 +120,7 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres string[] subDirs = file.RemotePath.Split('/'); string parentDir = string.Join("/", subDirs.Take(subDirs.Length - 1)); - if(!await _client.ExistsAsync(parentDir)) await CreateDirectoryAsync(parentDir); + if (!await _client.ExistsAsync(parentDir)) await CreateDirectoryAsync(parentDir); await using SftpFileStream createStream = _client.Create(file.RemotePath); await using FileStream openStream = File.OpenRead(file.LocalPath); From d669c5b97e04b7a72ee5d49d1d75143544649b7b Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 07:18:32 -0600 Subject: [PATCH 13/17] Fixed #36 --- Parallel.Cli/Commands/PushCommand.cs | 14 +++++++------- Parallel.Cli/Commands/VaultsCommand.cs | 14 ++++++++------ Parallel.Core/IO/Syncing/BaseSyncManager.cs | 2 +- Parallel.Core/IO/Syncing/DeltaSyncManager.cs | 2 +- Parallel.Core/IO/Syncing/FileSyncManager.cs | 2 +- Parallel.Core/IO/Syncing/ISyncManager.cs | 3 ++- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 15 +++++++-------- Parallel.Core/Settings/ParallelConfig.cs | 2 +- 8 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index 33e7b4d..e53c951 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -22,14 +22,14 @@ public class PushCommand : Command private readonly Option _sourceArg = new(["--path", "-p"], "The source path to sync."); private readonly Option _configOpt = new(["--config", "-c"], "The vault configuration to use."); - private readonly Option _verboseOpt = new(["--verbose", "-v"], "Shows verbose output."); + private readonly Option _forceOpt = new(["--force", "-f"], "Forces the pull overwriting any files."); public PushCommand() : base("push", "Pushes changed files to vaults.") { this.AddOption(_sourceArg); this.AddOption(_configOpt); - this.AddOption(_verboseOpt); - this.SetHandler(async (path, config, verbose) => + this.AddOption(_forceOpt); + this.SetHandler(async (path, config, force) => { _sw = Stopwatch.StartNew(); if (string.IsNullOrEmpty(path)) @@ -38,10 +38,10 @@ public PushCommand() : base("push", "Pushes changed files to vaults.") } else { - await SyncPathAsync(path); + await SyncPathAsync(path, force); } - }, _sourceArg, _configOpt, _verboseOpt); + }, _sourceArg, _configOpt, _forceOpt); } private Task SyncSystemAsync() @@ -49,7 +49,7 @@ private Task SyncSystemAsync() throw new NotImplementedException(); } - private async Task SyncPathAsync(string path) + private async Task SyncPathAsync(string path, bool force) { CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); await Program.Settings.ForEachVaultAsync(async vault => @@ -93,7 +93,7 @@ await Program.Settings.ForEachVaultAsync(async vault => } CommandLine.WriteLine(vault, $"Backing up {files.Length.ToString("N0")} files...", ConsoleColor.DarkGray); - await syncManager.PushFilesAsync(files, new ProgressReport(vault, successFiles)); + await syncManager.PushFilesAsync(files, force, new ProgressReport(vault, successFiles)); //await syncManager.PushFilesAsync(files, new ProgressBarReporter()); await syncManager.DisconnectAsync(); diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index 6851a69..c524f4d 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -17,11 +17,11 @@ public class VaultsCommand : Command private readonly Option configOpt = new(["--config", "-c"], "The vault configuration to use."); private readonly Command addCmd = new("add", "Adds a new vault configuration."); - private Command editCmd = new("edit", "Edits a vault configuration."); + private readonly Command editCmd = new("edit", "Edits a vault configuration."); private readonly Command findCmd = new("find", "Finds vault configurations in a location."); private readonly Command viewCmd = new("view", "Shows the vault configuration."); private readonly Command setCmd = new("set", "Sets a new vault configuration."); - private Command delCmd = new("delete", "Deletes a vault configuration."); + private readonly Command delCmd = new("delete", "Deletes a vault configuration."); public VaultsCommand() : base("vaults", "View or edit the vaults.") { @@ -44,10 +44,9 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") Service = Enum.Parse(CommandLine.ReadString($"Service ({string.Join(", ", Enum.GetNames(typeof(FileService)))})") ?? string.Empty, true) }; - string profileId = CommandLine.ReadString("Id") ?? HashGenerator.GenerateHash(8, true); - string profileName = CommandLine.ReadString("Name") ?? "Default"; if (spc.Service == FileService.Local) { + CommandLine.WriteLine("It is NOT RECOMMENDED to use a network drive!", ConsoleColor.Yellow); spc.RootDirectory = CommandLine.ReadString("Root") ?? string.Empty; } else if (spc.Service == FileService.Cloud) @@ -64,8 +63,11 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") spc.Password = CommandLine.ReadPassword("Password"); } - //spc.Encrypt = CommandLine.ReadBool("Encrypt files? (y/n)", false); - //spc.EncryptionKey = HashGenerator.GenerateHash(32, true); + string profileId = CommandLine.ReadString("Id") ?? HashGenerator.GenerateHash(8, true); + string profileName = CommandLine.ReadString("Name") ?? "Default"; + + spc.Encrypt = CommandLine.ReadBool("Encrypt files? (y/n)", false); + spc.EncryptionKey = spc.Encrypt ? HashGenerator.GenerateHash(32, true) : null; LocalVaultConfig localVault = new(profileId, profileName, spc); Program.Settings.Vaults.Add(localVault); diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index 95b7ae9..0bb3e9c 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -98,7 +98,7 @@ public async Task DisconnectAsync() } /// - public abstract Task PushFilesAsync(SystemFile[] files, IProgressReporter progress); + public abstract Task PushFilesAsync(SystemFile[] files, bool force, IProgressReporter progress); /// public abstract Task PullFilesAsync(SystemFile[] files, IProgressReporter progress); diff --git a/Parallel.Core/IO/Syncing/DeltaSyncManager.cs b/Parallel.Core/IO/Syncing/DeltaSyncManager.cs index be0b55c..56551f2 100644 --- a/Parallel.Core/IO/Syncing/DeltaSyncManager.cs +++ b/Parallel.Core/IO/Syncing/DeltaSyncManager.cs @@ -18,7 +18,7 @@ public class DeltaSyncManager : BaseSyncManager public DeltaSyncManager(RemoteVaultConfig remoteVaultConfig) : base(remoteVaultConfig) { } /// - public override Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) + public override Task PushFilesAsync(SystemFile[] files, bool force, IProgressReporter progress) { throw new NotImplementedException(); } diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 5a9ae31..cc76941 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -19,7 +19,7 @@ public class FileSyncManager : BaseSyncManager public FileSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// - public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) + public override async Task PushFilesAsync(SystemFile[] files, bool force, IProgressReporter progress) { if (!files.Any()) return; SystemFile[] backupFiles = files.Where(f => !f.Deleted).ToArray(); diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index 315bfff..c77d455 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -48,8 +48,9 @@ public interface ISyncManager /// Pushes an array of files to a vault. /// /// + /// /// - Task PushFilesAsync(SystemFile[] files, IProgressReporter progress); + Task PushFilesAsync(SystemFile[] files, bool force, IProgressReporter progress); /// /// Pulls an array of files from a vault. diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index c5d9c0f..b17ba5e 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -3,7 +3,6 @@ using System.Buffers; using System.Collections.Concurrent; using System.Diagnostics; -using System.Reflection.Metadata; using System.Threading.Channels; using Newtonsoft.Json.Linq; using Parallel.Core.Database; @@ -11,7 +10,6 @@ using Parallel.Core.Models; using Parallel.Core.Security; using Parallel.Core.Settings; -using Parallel.Core.Utils; using Parallel.Core.Workers; namespace Parallel.Core.IO.Syncing @@ -35,7 +33,7 @@ public class ObjectSyncManager : BaseSyncManager public ObjectSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// - public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) + public override async Task PushFilesAsync(SystemFile[] files, bool force, IProgressReporter progress) { long queued = 0, completed = 0, total = 0; TimeSpan uploadTimeout = TimeSpan.FromSeconds(30); @@ -58,9 +56,11 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter { Interlocked.Increment(ref total); string fullPath = PathBuilder.Combine(job.RemotePath, job.Filename); + if (await StorageProvider.ExistsAsync(fullPath)) { Log.Debug($"[WORKER {workerId}] UPLOAD SKIPPED: {fullPath}"); + Interlocked.Increment(ref completed); continue; } @@ -106,7 +106,6 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options return; } - await Database.AddFileAsync(file); await using FileStream fs = new FileStream(file.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: ChunkSize, useAsync: true); byte[] buffer = ArrayPool.Shared.Rent(ChunkSize); @@ -123,10 +122,9 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string basePath = PathBuilder.Combine(RemoteVault.Credentials.RootDirectory, "Parallel", RemoteVault.Id, "objects"); string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); - string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - - UploadWorker worker = new(chunk, parentDir, hash[4..], ex => progress.Failed(ex, file)); + string remotePath = PathBuilder.Combine(parentDir, hash); + UploadWorker worker = new(chunk, hash, parentDir, ex => progress.Failed(ex, file)); Task writeTask = channel.Writer.WriteAsync(worker, ct).AsTask(); Task timeoutTask = await Task.WhenAny(writeTask, Task.Delay(writeBlockWarn, ct)); if (timeoutTask != writeTask) @@ -136,7 +134,6 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } Interlocked.Increment(ref queued); - progress.Report(ProgressOperation.Pushed, file); } } finally @@ -144,7 +141,9 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options ArrayPool.Shared.Return(buffer); } + progress.Report(ProgressOperation.Pushed, file); await Database.AddHistoryAsync(file.LocalPath, HistoryType.Pushed); + await Database.AddFileAsync(file); } catch (Exception ex) { diff --git a/Parallel.Core/Settings/ParallelConfig.cs b/Parallel.Core/Settings/ParallelConfig.cs index 856deae..0f89d30 100644 --- a/Parallel.Core/Settings/ParallelConfig.cs +++ b/Parallel.Core/Settings/ParallelConfig.cs @@ -81,7 +81,7 @@ public class ParallelConfig /// The profiles to use. /// When pulling, the CLI defaults to the first in the list. /// - public HashSet Vaults { get; } = []; + public HashSet Vaults { get; set; } = []; /// From a8a63d5c662a70e8b3014345e8df6181170e9c3d Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 07:27:10 -0600 Subject: [PATCH 14/17] Added --force option for push overwrites --- Parallel.Cli/Commands/PushCommand.cs | 2 +- Parallel.Core/IO/Scanning/FileScanner.cs | 5 +++-- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index e53c951..a3a85fd 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -83,7 +83,7 @@ await Program.Settings.ForEachVaultAsync(async vault => CommandLine.WriteLine(vault, $"Scanning for file changes in {path}...", ConsoleColor.DarkGray); FileScanner scanner = new FileScanner(syncManager); - SystemFile[] files = await scanner.GetFileChangesAsync(path, ignoredFolders); + SystemFile[] files = await scanner.GetFileChangesAsync(path, ignoredFolders, force); int successFiles = files.Length; if (successFiles == 0) { diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index 4916b34..0bc6445 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -52,8 +52,9 @@ public SystemFile[] GetFileChanges() /// /// /// + /// /// A list of files that have changed since the last backup. - public async Task GetFileChangesAsync(string path, string[] ignoreFolders) + public async Task GetFileChangesAsync(string path, string[] ignoreFolders, bool force) { if (!Directory.Exists(path)) return Array.Empty(); @@ -77,7 +78,7 @@ public async Task GetFileChangesAsync(string path, string[] ignore localFile.Deleted = true; changedFiles.Add(localFile); } - else if (HasChanged(localFile, remoteFile)) + else if (HasChanged(localFile, remoteFile) || force) { Log.Debug($"Changed -> {localFile.LocalPath}"); localFile.RemotePath = remoteFile.RemotePath; diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index b17ba5e..a6b2276 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -57,7 +57,7 @@ public override async Task PushFilesAsync(SystemFile[] files, bool force, IProgr Interlocked.Increment(ref total); string fullPath = PathBuilder.Combine(job.RemotePath, job.Filename); - if (await StorageProvider.ExistsAsync(fullPath)) + if (await StorageProvider.ExistsAsync(fullPath) && !force) { Log.Debug($"[WORKER {workerId}] UPLOAD SKIPPED: {fullPath}"); Interlocked.Increment(ref completed); From c6eedb8b9b02eb7468f89914b59f49ea1c995eaa Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 08:02:07 -0600 Subject: [PATCH 15/17] Fixed merge conflicts with PR #38 --- Parallel.Core/Storage/StorageConnection.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Parallel.Core/Storage/StorageConnection.cs b/Parallel.Core/Storage/StorageConnection.cs index 3e0e251..6798286 100644 --- a/Parallel.Core/Storage/StorageConnection.cs +++ b/Parallel.Core/Storage/StorageConnection.cs @@ -29,11 +29,7 @@ public enum FileService /// /// Represents the way to connect to different file system associations. This class cannot be inherited. /// -<<<<<<<< HEAD:Parallel.Core/Storage/StorageConnection.cs public static class StorageConnection -======== - public static class StorageProvider ->>>>>>>> 69571e55dfea132b09b376c68487350de8bb3589:Parallel.Core/Storage/StorageProvider.cs { /// /// Creates a new file system association. From 5bf812628f67813762509f496a100f40c0c453d1 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 08:11:59 -0600 Subject: [PATCH 16/17] Fixed dotnet format issue --- Parallel.Cli/Utils/CommandLine.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Parallel.Cli/Utils/CommandLine.cs b/Parallel.Cli/Utils/CommandLine.cs index fed2b37..653f971 100644 --- a/Parallel.Cli/Utils/CommandLine.cs +++ b/Parallel.Cli/Utils/CommandLine.cs @@ -7,7 +7,10 @@ namespace Parallel.Cli.Utils { - public class CommandLine + /// + /// Represents a series of methods for printing to the console. This class can not be inherited. + /// + public abstract class CommandLine { private static readonly object _consoleLock = new(); From 7945379b35c9763153f0f702fe1ac61d12b4930f Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 12 Dec 2025 08:18:39 -0600 Subject: [PATCH 17/17] Fixed formatting issue --- Parallel.Cli/Utils/CommandLine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parallel.Cli/Utils/CommandLine.cs b/Parallel.Cli/Utils/CommandLine.cs index 653f971..def0982 100644 --- a/Parallel.Cli/Utils/CommandLine.cs +++ b/Parallel.Cli/Utils/CommandLine.cs @@ -81,7 +81,7 @@ public static void Write(object value, ConsoleColor color = ConsoleColor.Gray) public static void WriteLine(LocalVaultConfig localVault, object value, ConsoleColor color = ConsoleColor.Gray) { string baseLog = $"[{localVault.Id}] {value}"; - switch(color) + switch (color) { default: Log.Information(baseLog);