From d40ce5c30711c889605fdb18023e96648eb76246 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Fri, 5 Dec 2025 07:31:35 -0600 Subject: [PATCH 1/6] Testing blob chunking --- Parallel.Core/IO/PathBuilder.cs | 8 +++++--- Parallel.Core/IO/Syncing/BlobSyncManager.cs | 9 +++++---- Parallel.Core/IO/Syncing/SyncManager.cs | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index b2b12fd..7bdc981 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -168,10 +168,12 @@ public static bool IsFile(string path) return !Directory.Exists(path) && File.Exists(path); } - public static string GetObjectPath(string basePath, string hash) + public static string GetObjectPath(string basePath, string hash, int shards = 3) { - if (hash.Length < 8) throw new ArgumentException("Hash too short for sharding", nameof(hash)); - return Path.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash.Substring(4, 2), hash.Substring(6, 2), hash); + if (hash.Length < shards * 2) throw new ArgumentException("Hash too short for sharding", nameof(hash)); + string parentDir = Path.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash.Substring(4, 2), hash.Substring(6, 2)); + if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); + return Path.Combine(parentDir, hash); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/BlobSyncManager.cs b/Parallel.Core/IO/Syncing/BlobSyncManager.cs index 7ae3b69..214649d 100644 --- a/Parallel.Core/IO/Syncing/BlobSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BlobSyncManager.cs @@ -1,6 +1,7 @@ // Copyright 2025 Kyle Ebbinga using System.Reflection.Metadata; +using Parallel.Core.Database; using Parallel.Core.Diagnostics; using Parallel.Core.IO.Blobs; using Parallel.Core.Models; @@ -29,15 +30,15 @@ public BlobSyncManager(LocalVaultConfig localVault) : base(localVault) /// public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) { - IEnumerable hashes = await _blobStorage.ChunkFileAsync(files.First().LocalPath, Path.Combine(TempDirectory, "objects"), new ProgressLogger()); - File.WriteAllText(_hashes, JsonConvert.SerializeObject(hashes, Formatting.Indented)); + await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + { + await _blobStorage.ChunkFileAsync(file.LocalPath, Path.Combine(TempDirectory, "objects"), new ProgressLogger()); + }); } /// public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter progress) { - IEnumerable? hashes = JsonConvert.DeserializeObject>(await File.ReadAllTextAsync(_hashes)); - await _blobStorage.AssembleFileAsync(hashes, Path.Combine(TempDirectory, "objects"), files.First().LocalPath); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/SyncManager.cs b/Parallel.Core/IO/Syncing/SyncManager.cs index 9151632..9d8b84a 100644 --- a/Parallel.Core/IO/Syncing/SyncManager.cs +++ b/Parallel.Core/IO/Syncing/SyncManager.cs @@ -16,7 +16,7 @@ public static class SyncManager /// public static ISyncManager CreateNew(LocalVaultConfig localVault) { - return new FileSyncManager(localVault); + return new BlobSyncManager(localVault); } } } \ No newline at end of file From e74ae01b72188de1905ca0c03d9fa1f0c9ea948f Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Sat, 6 Dec 2025 03:41:01 -0600 Subject: [PATCH 2/6] Pushing works for blobs --- .../Database/Contexts/SqliteContext.cs | 50 ++++++++++++++----- Parallel.Core/Database/IDatabase.cs | 6 ++- .../Diagnostics/NullProgressReporter.cs | 24 +++++++++ Parallel.Core/IO/Blobs/BlobStorage.cs | 26 +++------- Parallel.Core/IO/Blobs/FileManifest.cs | 22 ++++++++ .../IO/FileSystem/DotNetFileSystem.cs | 30 ++++------- Parallel.Core/IO/FileSystem/IFileSystem.cs | 15 +++--- Parallel.Core/IO/FileSystem/SftpFileSystem.cs | 16 +++++- Parallel.Core/IO/PathBuilder.cs | 7 ++- Parallel.Core/IO/Scanning/FileScanner.cs | 42 +++++++++------- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 3 ++ Parallel.Core/IO/Syncing/BlobSyncManager.cs | 39 ++++++++++++--- Parallel.Core/IO/Syncing/FileSyncManager.cs | 2 +- Parallel.Core/Models/Manifest.cs | 21 ++++++++ Parallel.Core/Models/SystemFile.cs | 4 +- Parallel.Core/Security/HashGenerator.cs | 9 +++- 16 files changed, 217 insertions(+), 99 deletions(-) create mode 100644 Parallel.Core/Diagnostics/NullProgressReporter.cs create mode 100644 Parallel.Core/IO/Blobs/FileManifest.cs create mode 100644 Parallel.Core/Models/Manifest.cs diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index dc74ff8..648959c 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -45,20 +45,34 @@ public async Task InitializeAsync() Log.Information("Creating index database..."); using IDbConnection connection = CreateConnection(); - await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` TEXT, PRIMARY KEY(`id`));"); + await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `chunks` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `manifestId`, `hash` TEXT NOT NULL, order_index INTEGER NOT NULL, FOREIGN KEY (manifestId) REFERENCES manifests(id));"); + await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `manifests` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` TEXT NOT NULL, `path` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL);"); + await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` BLOB, PRIMARY KEY(`id`));"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `history` (`timestamp` LONG INTEGER NOT NULL, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`timestamp`));"); } #endregion + #region Chunks + + /// + public async Task AddChunkAsync(int manifestId, string hash, int index) + { + using IDbConnection connection = CreateConnection(); + string sql = @"INSERT OR REPLACE INTO chunks (manifestId, hash, order_index) VALUES (@manifestId, @hash, @index);"; + return await connection.ExecuteAsync(sql, new { manifestId, hash, index }); + } + + #endregion + #region Files /// - public async Task AddFileAsync(SystemFile file) + public async Task AddFileAsync(SystemFile file) { using IDbConnection connection = CreateConnection(); string sql = @"INSERT OR REPLACE INTO files (id, name, localpath, remotepath, lastwrite, lastupdate, LocalSize, RemoteSize, type, hidden, readonly, deleted, checksum) VALUES (@Id, @Name, @LocalPath, @RemotePath, @LastWrite, @LastUpdate, @LocalSize, @RemoteSize, @Type, @Hidden, @ReadOnly, @Deleted, @CheckSum);"; - return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.CheckSum }) > 0; + return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.CheckSum }); } public async Task GetLocalSizeAsync() @@ -86,16 +100,16 @@ public async Task GetTotalFilesAsync(bool deleted) public async Task> GetFilesAsync(string path) { using IDbConnection connection = CreateConnection(); - string sql = $"SELECT * FROM files WHERE localpath LIKE \"%{path}%\" OR remotepath LIKE \"%{path}%\" ORDER BY lastupdate DESC"; - return await connection.QueryAsync(sql); + string sql = $"SELECT * FROM files WHERE localpath LIKE \"%@path%\" OR remotepath LIKE \"%@path%\" ORDER BY lastupdate DESC"; + return await connection.QueryAsync(sql, new { path }); } /// public async Task> GetFilesAsync(string path, bool deleted) { using IDbConnection connection = CreateConnection(); - string sql = $"SELECT * FROM files WHERE deleted = {deleted} ORDER BY lastupdate DESC"; - return await connection.QueryAsync(sql); + string sql = $"SELECT * FROM files WHERE deleted = @deleted ORDER BY lastupdate DESC"; + return await connection.QueryAsync(sql,new { deleted }); } /// @@ -111,13 +125,11 @@ public async Task> GetFilesAsync(string path, bool delet #region History /// - public async Task AddHistoryAsync(string path, HistoryType type) + public async Task AddHistoryAsync(string path, HistoryType type) { - using (IDbConnection connection = CreateConnection()) - { - string sql = @"INSERT OR REPLACE INTO history (timestamp, name, path, type) VALUES(@Timestamp, @Name, @Path, @Type);"; - return await connection.ExecuteAsync(sql, new { Timestamp = UnixTime.Now.TotalMilliseconds, Name = Path.GetFileName(path), Path = path, Type = type }) > 0; - } + using IDbConnection connection = CreateConnection(); + string sql = @"INSERT OR REPLACE INTO history (timestamp, name, path, type) VALUES(@Timestamp, @Name, @Path, @Type);"; + return await connection.ExecuteAsync(sql, new { Timestamp = UnixTime.Now.TotalMilliseconds, Name = Path.GetFileName(path), Path = path, Type = type }); } /// @@ -133,5 +145,17 @@ public async Task AddHistoryAsync(string path, HistoryType type) } #endregion + + #region Manifests + + /// + public async Task AddManifestAsync(SystemFile file) + { + using IDbConnection connection = CreateConnection(); + string sql = @"INSERT OR REPLACE INTO manifests (name, path, lastwrite) VALUES (@Name, @Path, @LastWrite);"; + return await connection.ExecuteAsync(sql, new { file.Name, Path = file.LocalPath, LastWrite = file.LastWrite.TotalMilliseconds }); + } + + #endregion } } \ No newline at end of file diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 4564eb8..7444709 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -72,7 +72,7 @@ public interface IDatabase : IDisposable /// /// /// True if successful, false otherwise - Task AddFileAsync(SystemFile file); + Task AddFileAsync(SystemFile file); Task GetLocalSizeAsync(); Task GetRemoteSizeAsync(); @@ -88,7 +88,7 @@ public interface IDatabase : IDisposable /// /// /// True if successful, false otherwise - Task AddHistoryAsync(string path, HistoryType type); + Task AddHistoryAsync(string path, HistoryType type); IEnumerable? GetHistory(string path, int limit); @@ -99,5 +99,7 @@ public interface IDatabase : IDisposable Task> GetFilesAsync(string path); Task> GetFilesAsync(string path, bool deleted); Task GetFileAsync(string path); + Task AddManifestAsync(SystemFile file); + Task AddChunkAsync(int manifestId, string hash, int length); } } \ No newline at end of file diff --git a/Parallel.Core/Diagnostics/NullProgressReporter.cs b/Parallel.Core/Diagnostics/NullProgressReporter.cs new file mode 100644 index 0000000..3402b20 --- /dev/null +++ b/Parallel.Core/Diagnostics/NullProgressReporter.cs @@ -0,0 +1,24 @@ +// Copyright 2025 Kyle Ebbinga + +using Parallel.Core.Models; + +namespace Parallel.Core.Diagnostics +{ + /// + /// Represents a null . + /// + public class NullProgressReporter : IProgressReporter + { + /// + public void Report(ProgressOperation operation, SystemFile file) { } + + /// + public void Reset() { } + + /// + public void Failed(Exception exception, SystemFile file) + { + Log.Error($"{exception.GetType().FullName}: {exception.Message}"); + } + } +} \ No newline at end of file diff --git a/Parallel.Core/IO/Blobs/BlobStorage.cs b/Parallel.Core/IO/Blobs/BlobStorage.cs index 630a143..668a8a3 100644 --- a/Parallel.Core/IO/Blobs/BlobStorage.cs +++ b/Parallel.Core/IO/Blobs/BlobStorage.cs @@ -1,6 +1,7 @@ // Copyright 2025 Kyle Ebbinga using Parallel.Core.Diagnostics; +using Parallel.Core.IO.FileSystem; using Parallel.Core.Security; namespace Parallel.Core.IO.Blobs @@ -8,32 +9,21 @@ namespace Parallel.Core.IO.Blobs /// /// Represents the way to chunk files into blobs for syncing. /// - public class BlobStorage + public abstract class BlobStorage { /// /// The size, in bytes, to use for chunks of a file. /// - public int ChunkSize { get; set; } - - /// - /// Gets the temp directory for storing blobs. - /// - public string TempDirectory { get; set; } - - public BlobStorage(string tempDir, int chunkSize = 4194304) - { - TempDirectory = tempDir; - ChunkSize = chunkSize; - } + private static readonly int ChunkSize = 4194304; /// /// Chunks a file into hashes for blob storage. /// /// The source path of the file. - /// The destination to send chunked objects to. + /// The temp directory to send chunked objects to. /// /// - public async Task> ChunkFileAsync(string sourcePath, string destPath, IProgressReporter progress) + public static async Task CreateManifestAsync(IFileSystem fileSystem, string sourcePath, string tempObjDir, IProgressReporter progress) { List chunkHashes = new List(); await using FileStream fs = File.OpenRead(sourcePath); @@ -46,14 +36,14 @@ public async Task> ChunkFileAsync(string sourcePath, string Buffer.BlockCopy(buffer, 0, chunkData, 0, bytesRead); string hash = HashGenerator.CreateSHA256(chunkData); - string chunkPath = PathBuilder.GetObjectPath(destPath, hash); + string chunkPath = PathBuilder.GetObjectPath(tempObjDir, hash); if(!File.Exists(chunkPath)) await File.WriteAllBytesAsync(chunkPath, chunkData); chunkHashes.Add(hash); } - Log.Debug($"Wrote {chunkHashes.Count} hashes to {destPath}"); - return chunkHashes; + Log.Debug($"Wrote {chunkHashes.Count} hashes to {tempObjDir}"); + return new FileManifest(sourcePath, chunkHashes, new FileInfo(sourcePath).Length); } /// diff --git a/Parallel.Core/IO/Blobs/FileManifest.cs b/Parallel.Core/IO/Blobs/FileManifest.cs new file mode 100644 index 0000000..d2afcec --- /dev/null +++ b/Parallel.Core/IO/Blobs/FileManifest.cs @@ -0,0 +1,22 @@ +// Copyright 2025 Kyle Ebbinga + +using Parallel.Core.Security; + +namespace Parallel.Core.IO.Blobs +{ + public class FileManifest + { + public string Id { get; } + public string Fullname { get; set; } + public List Hashes { get; } + public long Length { get; set; } + + public FileManifest(string path, IEnumerable hashes, long length) + { + Id = HashGenerator.CreateSHA1(path); + Fullname = path; + Hashes = hashes.ToList(); + Length = length; + } + } +} \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs index 799f181..05d6f6d 100644 --- a/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/DotNetFileSystem.cs @@ -5,6 +5,7 @@ using Microsoft.VisualBasic.FileIO; using Newtonsoft.Json.Linq; using Parallel.Core.Diagnostics; +using Parallel.Core.IO.Blobs; using Parallel.Core.Models; using Parallel.Core.Settings; using Parallel.Core.Utils; @@ -70,12 +71,11 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } /// - public async Task DownloadFileAsync(string sourcePath, string destinationPath) + public async Task DownloadStreamAsync(Stream output, string remotePath) { - await using FileStream openStream = File.OpenRead(destinationPath); - await using FileStream createStream = File.Create(sourcePath); + await using FileStream openStream = File.OpenRead(remotePath); await using GZipStream gzipStream = new GZipStream(openStream, CompressionMode.Decompress); - await gzipStream.CopyToAsync(createStream); + await gzipStream.CopyToAsync(output); } /// @@ -126,25 +126,13 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } /// - public async Task UploadFileAsync(string sourcePath, string destinationPath) + public async Task UploadStreamAsync(Stream input, string remotePath) { - try - { - if (await ExistsAsync(destinationPath)) File.SetAttributes(destinationPath, ~FileAttributes.ReadOnly & File.GetAttributes(destinationPath)); - string? parent = Path.GetDirectoryName(destinationPath); - if (parent != null && !Directory.Exists(parent)) Directory.CreateDirectory(parent); - - await using FileStream openStream = File.OpenRead(sourcePath); - await using FileStream createStream = File.Create(destinationPath); - await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); - await openStream.CopyToAsync(gzipStream); + await using FileStream createStream = File.Create(remotePath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await input.CopyToAsync(gzipStream); - File.SetAttributes(destinationPath, File.GetAttributes(destinationPath) | FileAttributes.ReadOnly); - } - catch (Exception ex) - { - Log.Error(ex.GetBaseException().ToString()); - } + //File.SetAttributes(remotePath, File.GetAttributes(remotePath) | FileAttributes.ReadOnly); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/IFileSystem.cs b/Parallel.Core/IO/FileSystem/IFileSystem.cs index 76ebe7f..74e6f8e 100644 --- a/Parallel.Core/IO/FileSystem/IFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/IFileSystem.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading.Tasks; using Parallel.Core.Diagnostics; +using Parallel.Core.IO.Blobs; using Parallel.Core.Models; namespace Parallel.Core.IO.FileSystem @@ -43,9 +44,9 @@ public interface IFileSystem : IDisposable /// /// Downloads a file from the associated file system. /// - /// - /// - Task DownloadFileAsync(string sourcePath, string destinationPath); + /// + /// + Task DownloadStreamAsync(Stream output, string remotePath); /// /// Checks if a path exists on the associated file system. @@ -69,10 +70,10 @@ public interface IFileSystem : IDisposable Task UploadFilesAsync(SystemFile[] files, IProgressReporter progress); /// - /// Uploads a file to the associated file system. + /// Uploads a stream to the associated file system. /// - /// - /// - Task UploadFileAsync(string sourcePath, string destinationPath); + /// + /// + Task UploadStreamAsync(Stream input, string remotePath); } } \ No newline at end of file diff --git a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs index 64ae552..da67727 100644 --- a/Parallel.Core/IO/FileSystem/SftpFileSystem.cs +++ b/Parallel.Core/IO/FileSystem/SftpFileSystem.cs @@ -7,6 +7,7 @@ using System.IO.Compression; using Newtonsoft.Json.Linq; using Parallel.Core.Diagnostics; +using Parallel.Core.IO.Blobs; using Parallel.Core.Models; using Parallel.Core.Security; using Parallel.Core.Utils; @@ -88,9 +89,11 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } /// - public Task DownloadFileAsync(string sourcePath, string destinationPath) + public async Task DownloadStreamAsync(Stream output, string remotePath) { - throw new NotImplementedException(); + await using SftpFileStream openStream = _client.OpenRead(remotePath); + await using GZipStream gzipStream = new GZipStream(openStream, CompressionMode.Decompress); + await gzipStream.CopyToAsync(output); } /// @@ -138,6 +141,15 @@ public async Task UploadFilesAsync(SystemFile[] files, IProgressReporter progres } } + /// + public async Task UploadStreamAsync(Stream input, string remotePath) + { + await using SftpFileStream createStream = _client.Create(remotePath); + await using GZipStream gzipStream = new GZipStream(createStream, CompressionLevel.SmallestSize); + await input.CopyToAsync(gzipStream); + //_client.ChangePermissions(remotePath, 444); + } + /// public async Task UploadFileAsync(string sourcePath, string destinationPath) { diff --git a/Parallel.Core/IO/PathBuilder.cs b/Parallel.Core/IO/PathBuilder.cs index 7bdc981..55ad1c3 100644 --- a/Parallel.Core/IO/PathBuilder.cs +++ b/Parallel.Core/IO/PathBuilder.cs @@ -168,12 +168,11 @@ public static bool IsFile(string path) return !Directory.Exists(path) && File.Exists(path); } - public static string GetObjectPath(string basePath, string hash, int shards = 3) + public static string GetObjectPath(string basePath, string hash) { - if (hash.Length < shards * 2) throw new ArgumentException("Hash too short for sharding", nameof(hash)); - string parentDir = Path.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash.Substring(4, 2), hash.Substring(6, 2)); + string parentDir = Path.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); - return Path.Combine(parentDir, hash); + return Path.Combine(parentDir, hash[4..]); } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index f4a210d..8b09b42 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -1,5 +1,6 @@ // Copyright 2025 Kyle Ebbinga +using System.Collections.Concurrent; using System.Data; using System.Diagnostics; using System.Text; @@ -54,51 +55,57 @@ public async Task GetFileChangesAsync(string path, string[] ignore { if (!Directory.Exists(path)) return Array.Empty(); - List scannedFiles = new List(); + ConcurrentBag scannedFiles = new(); + ConcurrentBag changedFiles = new(); + HashSet localFiles = FileScanner.GetFiles(path, ignoreFolders, ".").ToHashSet(); IEnumerable remoteFiles = await _db.GetFilesAsync(path, false); - await System.Threading.Tasks.Parallel.ForEachAsync(remoteFiles, ParallelConfig.Options, async (remoteFile, ct) => + System.Threading.Tasks.Parallel.ForEach(remoteFiles, ParallelConfig.Options, (remoteFile, ct) => { if (File.Exists(remoteFile.LocalPath)) { SystemFile localFile = new SystemFile(remoteFile.LocalPath); if (IsIgnored(localFile.LocalPath, ignoreFolders)) { - Log.Debug($"Ignored -> {localFile.LocalPath}"); + //Log.Debug($"Ignored -> {localFile.LocalPath}"); localFile.RemotePath = remoteFile.RemotePath; localFile.Deleted = true; - scannedFiles.Add(localFile); + changedFiles.Add(localFile); } else if (HasChanged(localFile, remoteFile)) { - Log.Debug($"Changed -> {localFile.LocalPath}"); + //Log.Debug($"Changed -> {localFile.LocalPath}"); localFile.RemotePath = remoteFile.RemotePath; - scannedFiles.Add(localFile); + changedFiles.Add(localFile); } - localFiles.Remove(localFile.LocalPath); + scannedFiles.Add(localFile.LocalPath); } else { - Log.Debug($"Deleted -> {remoteFile.LocalPath}"); + //Log.Debug($"Deleted -> {remoteFile.LocalPath}"); remoteFile.Deleted = true; - scannedFiles.Add(remoteFile); + changedFiles.Add(remoteFile); } }); - Log.Debug($"{localFiles.Count} files are untracked! Adding..."); - await System.Threading.Tasks.Parallel.ForEachAsync(localFiles, ParallelConfig.Options, async (file, ct) => + HashSet remainingFiles = localFiles.Except(new HashSet(scannedFiles)).ToHashSet(); + Log.Debug($"{remainingFiles.Count} files are untracked! Adding..."); + System.Threading.Tasks.Parallel.ForEach(remainingFiles, ParallelConfig.Options, (file, ct) => { - if (File.Exists(file) && !IsIgnored(file, ignoreFolders)) + if (!IsIgnored(file, ignoreFolders)) { - Log.Debug($"Created -> {file}"); - scannedFiles.Add(new SystemFile(file) { RemotePath = PathBuilder.Remote(file, _config) }); + if (File.Exists(file)) + { + //Log.Debug($"Created -> {file}"); + changedFiles.Add(new SystemFile(file) { RemotePath = PathBuilder.Remote(file, _config) }); + } } }); Log.Debug($"{localFiles.Count} files remaining."); - Log.Information($"Found {scannedFiles.Count:N0} changes in '{path}'"); - return scannedFiles.ToArray(); + Log.Information($"Found {changedFiles.Count:N0} changes in '{path}'"); + return changedFiles.ToArray(); } /// @@ -109,8 +116,6 @@ await System.Threading.Tasks.Parallel.ForEachAsync(localFiles, ParallelConfig.Op /// True is success, otherwise false. public static bool HasChanged(SystemFile sourcePath, SystemFile? targetPath) { - Console.WriteLine($"{sourcePath.Name}: {targetPath} == null || ({sourcePath.LastWrite.TotalMilliseconds} > {targetPath.LastWrite.TotalMilliseconds} && {!sourcePath.CheckSum.SequenceEqual(targetPath.CheckSum)}"); - return targetPath == null || (sourcePath.LastWrite.TotalMilliseconds > targetPath.LastWrite.TotalMilliseconds && !sourcePath.CheckSum.SequenceEqual(targetPath.CheckSum)); } @@ -223,7 +228,6 @@ public static IEnumerable GetFiles(string root, string[] exempt, string while (pending.Count > 0) { string current = pending.Pop(); - if (IsIgnored(current, exempt)) { Log.Debug($"Ignored -> {current}"); diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index 116060e..604a7af 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -88,6 +88,9 @@ public async Task ConnectAsync() /// public async Task DisconnectAsync() { + Log.Debug($"Uploaded config file: {TempConfigFile}"); + Log.Debug($"Uploaded db file: {TempDbFile}"); + SystemFile[] tempFiles = [new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault)), new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))]; await FileSystem.UploadFilesAsync(tempFiles, new ProgressLogger()); FileSystem.Dispose(); diff --git a/Parallel.Core/IO/Syncing/BlobSyncManager.cs b/Parallel.Core/IO/Syncing/BlobSyncManager.cs index 214649d..7c55dd3 100644 --- a/Parallel.Core/IO/Syncing/BlobSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BlobSyncManager.cs @@ -1,10 +1,12 @@ // Copyright 2025 Kyle Ebbinga using System.Reflection.Metadata; +using Newtonsoft.Json.Linq; using Parallel.Core.Database; using Parallel.Core.Diagnostics; using Parallel.Core.IO.Blobs; using Parallel.Core.Models; +using Parallel.Core.Security; using Parallel.Core.Settings; namespace Parallel.Core.IO.Syncing @@ -14,25 +16,46 @@ namespace Parallel.Core.IO.Syncing /// public class BlobSyncManager : BaseSyncManager { - private readonly BlobStorage _blobStorage; - private string _hashes; + /// + /// The size, in bytes, to use for chunks of a file. + /// + private static readonly int ChunkSize = 4194304; + /// /// Initializes a new instance of the class. /// /// - public BlobSyncManager(LocalVaultConfig localVault) : base(localVault) - { - _blobStorage = new BlobStorage(TempDirectory); - _hashes = Path.Combine(TempDirectory, "Hashes.json"); - } + public BlobSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) { await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - await _blobStorage.ChunkFileAsync(file.LocalPath, Path.Combine(TempDirectory, "objects"), new ProgressLogger()); + int bytesRead = 0; + byte[] buffer = new byte[ChunkSize]; + await using FileStream fs = File.OpenRead(file.LocalPath); + + int index = 0; + progress.Report(ProgressOperation.Uploading, file); + int row = await Database.AddManifestAsync(file); + while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) + { + await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); + string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); + await Database.AddChunkAsync(row, hash, index); + index++; + + string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); + string remotePath = PathBuilder.Combine(parentDir, hash[4..]); + if (!await FileSystem.ExistsAsync(remotePath)) + { + if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); + await FileSystem.UploadStreamAsync(ms, remotePath); + } + } }); } diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 5b0eabe..d069ad6 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -37,7 +37,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } else { - //progress.Report(ProgressOperation.Syncing, file); + progress.Report(ProgressOperation.Syncing, file); SystemFile? remote = await FileSystem.GetFileAsync(file.RemotePath); if (remote is not null) { diff --git a/Parallel.Core/Models/Manifest.cs b/Parallel.Core/Models/Manifest.cs new file mode 100644 index 0000000..2e59b33 --- /dev/null +++ b/Parallel.Core/Models/Manifest.cs @@ -0,0 +1,21 @@ +// Copyright 2025 Kyle Ebbinga + +using Parallel.Core.Security; + +namespace Parallel.Core.Models +{ + /// + /// Represents a manifest + /// + public class Manifest + { + public string Id { get; } + public string Fullname { get; } + + public Manifest(string path) + { + Id = HashGenerator.CreateSHA1(path); + Fullname = path; + } + } +} \ No newline at end of file diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index 9ad23ca..65cf1bd 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -76,7 +76,7 @@ public class SystemFile /// /// The checksum used to check if the file has changed. /// - public string? CheckSum { get; set; } + public byte[]? CheckSum { get; set; } /// @@ -124,7 +124,7 @@ public SystemFile(string localPath, string remotePath) /// /// /// - public SystemFile(string id, string name, string localpath, string remotepath, long lastwrite, long lastupdate, long localsize, long remotesize, string type, long hidden, long readOnly, long deleted, string checksum) + public SystemFile(string id, string name, string localpath, string remotepath, long lastwrite, long lastupdate, long localsize, long remotesize, string type, long hidden, long readOnly, long deleted, byte[] checksum) { Id = id; Name = name; diff --git a/Parallel.Core/Security/HashGenerator.cs b/Parallel.Core/Security/HashGenerator.cs index 6fda9c4..9f7c008 100644 --- a/Parallel.Core/Security/HashGenerator.cs +++ b/Parallel.Core/Security/HashGenerator.cs @@ -71,6 +71,11 @@ public static string CreateSHA1(string value) return Convert.ToHexString(SHA1.HashData(Encoding.ASCII.GetBytes(value))).ToLower(); } + public static string CreateSHA256(Span value) + { + return Convert.ToHexString(SHA256.HashData(value)).ToLower(); + } + /// /// Computes a SHA256 hash from bytes. /// @@ -97,12 +102,12 @@ public static string CreateSHA256(string value) /// /// /// - public static string? CheckSum(string path) + public static byte[]? CheckSum(string path) { if (!File.Exists(path)) return null; using FileStream fs = File.OpenRead(path); using SHA256 sha256 = SHA256.Create(); - return Convert.ToHexString(sha256.ComputeHash(fs)).ToLowerInvariant(); + return sha256.ComputeHash(fs); } } } \ No newline at end of file From e35adf9e6fab7dc7880bc859b857e77a42620698 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Sun, 7 Dec 2025 01:59:17 -0600 Subject: [PATCH 3/6] More work on blob syncing --- Parallel.Cli/Commands/CleanCommand.cs | 86 +++++++++---------- .../Database/Contexts/SqliteContext.cs | 2 +- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 2 +- Parallel.Core/IO/Syncing/BlobSyncManager.cs | 41 +++++---- 4 files changed, 69 insertions(+), 62 deletions(-) diff --git a/Parallel.Cli/Commands/CleanCommand.cs b/Parallel.Cli/Commands/CleanCommand.cs index 6b5cb35..b05a096 100644 --- a/Parallel.Cli/Commands/CleanCommand.cs +++ b/Parallel.Cli/Commands/CleanCommand.cs @@ -30,6 +30,7 @@ public CleanCommand() : base("clean", "Cleans up the file system by removing old { ParallelConfig config = ParallelConfig.Load(); if (days <= config.RetentionPeriod) days = config.RetentionPeriod; + CommandLine.WriteLine($"Scanning for cleanable files older than {days:N0} days old...", ConsoleColor.DarkGray); if (string.IsNullOrEmpty(path)) { @@ -54,72 +55,71 @@ await System.Threading.Tasks.Parallel.ForEachAsync(config.CleanDirectories, Para private async Task CleanDirectoryAsync(ParallelConfig config, string path, int days, bool recursive, bool verbose) { - CommandLine.WriteLine($"Scanning for cleanable files older than {days:N0} days old in {path}...", ConsoleColor.DarkGray); if (!Directory.Exists(path)) { - CommandLine.WriteLine($"The provided path was not found!", ConsoleColor.Yellow); + CommandLine.WriteLine($"Unable to find path: '{path}'", ConsoleColor.Yellow); return; } UnixTime minTime = UnixTime.FromMilliseconds(UnixTime.Now.TotalMilliseconds - (days * UnixTime.Day)); IEnumerable cleanableFiles = FileScanner.GetCleanableFiles(path, minTime, recursive); - if (!cleanableFiles.Any()) CommandLine.WriteLine($"No cleanable files were found in the provided path.", ConsoleColor.Green); - - await System.Threading.Tasks.Parallel.ForEachAsync(cleanableFiles, ParallelConfig.Options, (fi, ct) => + if (cleanableFiles.Any()) { - if (fi.Exists) + await System.Threading.Tasks.Parallel.ForEachAsync(cleanableFiles, ParallelConfig.Options, (fi, ct) => { - try - { - _freedBytes += fi.Length; - _filesCount++; - fi.Delete(); - } - catch (Exception ex) + if (fi.Exists) { - CommandLine.WriteLine($"Unable to remove file: {fi.FullName}", ConsoleColor.Yellow); - Log.Warning($"{ex.GetBaseException().Message}"); + try + { + _freedBytes += fi.Length; + _filesCount++; + fi.Delete(); + } + catch (Exception ex) + { + CommandLine.WriteLine($"Unable to remove file: {fi.FullName}", ConsoleColor.Yellow); + Log.Warning($"{ex.GetBaseException().Message}"); + } } - } - return ValueTask.CompletedTask; - }); + return ValueTask.CompletedTask; + }); + } IEnumerable directories = FileScanner.GetEmptyDirectories(path, recursive); - if (!directories.Any()) CommandLine.WriteLine($"No empty directories were found in the provided path.", ConsoleColor.Green); - - await System.Threading.Tasks.Parallel.ForEachAsync(directories, ParallelConfig.Options, (di, ct) => + if (!directories.Any()) { - if (di.Exists && !di.EnumerateFiles().Any()) + System.Threading.Tasks.Parallel.ForEach(directories, ParallelConfig.Options, (di) => + { + if (di.Exists && !di.EnumerateFiles().Any()) + { + try + { + Log.Debug($"Removing empty directory: {di?.FullName}"); + di?.Delete(true); + } + catch (Exception ex) + { + Log.Error($"{ex.GetBaseException().Message}"); + } + } + }); + + DirectoryInfo currentDir = new DirectoryInfo(path); + SearchOption option = recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; + if (!currentDir.EnumerateFiles("*", option).Any()) { try { - Log.Debug($"Removing empty directory: {di?.FullName}"); - di?.Delete(true); + Log.Debug($"Removing empty directory: {currentDir.FullName}"); + currentDir.Delete(true); + _dirsCount++; } catch (Exception ex) { - Log.Error($"{ex.GetBaseException().Message}"); + Log.Warning($"{ex.GetBaseException().Message}"); } } - - return ValueTask.CompletedTask; - }); - - DirectoryInfo currentDir = new DirectoryInfo(path); - SearchOption option = recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; - if (!currentDir.EnumerateFiles("*", option).Any()) - { - try - { - Log.Debug($"Removing empty directory: {currentDir.FullName}"); - currentDir.Delete(true); - _dirsCount++; - } - catch (Exception ex) - { - Log.Warning($"{ex.GetBaseException().Message}"); - } } } } diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 648959c..8356162 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -46,7 +46,7 @@ public async Task InitializeAsync() using IDbConnection connection = CreateConnection(); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `chunks` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `manifestId`, `hash` TEXT NOT NULL, order_index INTEGER NOT NULL, FOREIGN KEY (manifestId) REFERENCES manifests(id));"); - await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `manifests` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` TEXT NOT NULL, `path` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL);"); + await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `manifests` (`id` INTEGER AUTOINCREMENT, `name` TEXT NOT NULL, `path` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL`type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` BLOB, PRIMARY KEY (`id`, `path`));"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` BLOB, PRIMARY KEY(`id`));"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `history` (`timestamp` LONG INTEGER NOT NULL, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`timestamp`));"); } diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index 604a7af..8a54ba8 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -32,7 +32,7 @@ public abstract class BaseSyncManager : ISyncManager /// /// /// - /// + /// public BaseSyncManager(LocalVaultConfig localVault) { FileSystem = FileSystemManager.CreateNew(localVault); diff --git a/Parallel.Core/IO/Syncing/BlobSyncManager.cs b/Parallel.Core/IO/Syncing/BlobSyncManager.cs index 7c55dd3..b549da6 100644 --- a/Parallel.Core/IO/Syncing/BlobSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BlobSyncManager.cs @@ -33,27 +33,34 @@ public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter { await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - int bytesRead = 0; - byte[] buffer = new byte[ChunkSize]; - await using FileStream fs = File.OpenRead(file.LocalPath); + if (file.Deleted) + { - int index = 0; - progress.Report(ProgressOperation.Uploading, file); - int row = await Database.AddManifestAsync(file); - while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) + } + else { - await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); - string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); - await Database.AddChunkAsync(row, hash, index); - index++; + int bytesRead = 0; + byte[] buffer = new byte[ChunkSize]; + await using FileStream fs = File.OpenRead(file.LocalPath); - string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); - string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); - string remotePath = PathBuilder.Combine(parentDir, hash[4..]); - if (!await FileSystem.ExistsAsync(remotePath)) + int index = 0; + progress.Report(ProgressOperation.Uploading, file); + int row = await Database.AddManifestAsync(file); + while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) { - if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); - await FileSystem.UploadStreamAsync(ms, remotePath); + await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); + string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); + await Database.AddChunkAsync(row, hash, index); + index++; + + string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string parentDir = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2)); + string remotePath = PathBuilder.Combine(parentDir, hash[4..]); + if (!await FileSystem.ExistsAsync(remotePath)) + { + if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); + await FileSystem.UploadStreamAsync(ms, remotePath); + } } } }); From 45ed0f94e366520921e94926204d89fd600d2922 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Mon, 8 Dec 2025 19:24:02 -0600 Subject: [PATCH 4/6] Database now properly stores objects --- .../Database/Contexts/SqliteContext.cs | 45 ++++++++----------- Parallel.Core/Database/IDatabase.cs | 23 ++++++---- ...lobSyncManager.cs => ObjectSyncManager.cs} | 20 ++++++--- Parallel.Core/IO/Syncing/SyncManager.cs | 2 +- Parallel.Core/Models/SystemFile.cs | 10 +++-- 5 files changed, 54 insertions(+), 46 deletions(-) rename Parallel.Core/IO/Syncing/{BlobSyncManager.cs => ObjectSyncManager.cs} (76%) diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 8356162..4c24659 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using Dapper; using Parallel.Core.IO; +using Parallel.Core.IO.Blobs; using Parallel.Core.Models; using Parallel.Core.Settings; using Parallel.Core.Utils; @@ -26,11 +27,6 @@ public SqliteContext(string filePath) FilePath = filePath; } - public void Dispose() - { - // TODO release managed resources here - } - #region Base /// @@ -45,34 +41,21 @@ public async Task InitializeAsync() Log.Information("Creating index database..."); using IDbConnection connection = CreateConnection(); - await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `chunks` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `manifestId`, `hash` TEXT NOT NULL, order_index INTEGER NOT NULL, FOREIGN KEY (manifestId) REFERENCES manifests(id));"); - await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `manifests` (`id` INTEGER AUTOINCREMENT, `name` TEXT NOT NULL, `path` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL`type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` BLOB, PRIMARY KEY (`id`, `path`));"); + await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `objects` (`id` TEXT NOT NULL, `hash` TEXT NOT NULL, orderIndex INTEGER NOT NULL, UNIQUE (id, orderIndex));"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `files` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `localpath` TEXT NOT NULL, `remotepath` TEXT NOT NULL, `lastwrite` LONG INTEGER NOT NULL, `lastupdate` LONG INTEGER NOT NULL, `localsize` LONG INTEGER NOT NULL, `remotesize` LONG INTEGER NOT NULL, `type` TEXT NOT NULL DEFAULT Other CHECK(`type` IN ('Document', 'Photo', 'Music', 'Video', 'Other')), `hidden` INTEGER NOT NULL DEFAULT 0, `readonly` INTEGER NOT NULL DEFAULT 0, `deleted` INTEGER NOT NULL DEFAULT 0, `checksum` BLOB, PRIMARY KEY(`id`));"); await connection.ExecuteAsync("CREATE TABLE IF NOT EXISTS `history` (`timestamp` LONG INTEGER NOT NULL, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`timestamp`));"); } #endregion - #region Chunks - - /// - public async Task AddChunkAsync(int manifestId, string hash, int index) - { - using IDbConnection connection = CreateConnection(); - string sql = @"INSERT OR REPLACE INTO chunks (manifestId, hash, order_index) VALUES (@manifestId, @hash, @index);"; - return await connection.ExecuteAsync(sql, new { manifestId, hash, index }); - } - - #endregion - #region Files /// - public async Task AddFileAsync(SystemFile file) + public async Task AddFileAsync(SystemFile file) { using IDbConnection connection = CreateConnection(); string sql = @"INSERT OR REPLACE INTO files (id, name, localpath, remotepath, lastwrite, lastupdate, LocalSize, RemoteSize, type, hidden, readonly, deleted, checksum) VALUES (@Id, @Name, @LocalPath, @RemotePath, @LastWrite, @LastUpdate, @LocalSize, @RemoteSize, @Type, @Hidden, @ReadOnly, @Deleted, @CheckSum);"; - return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.CheckSum }); + return await connection.ExecuteAsync(sql, new { file.Id, file.Name, file.LocalPath, file.RemotePath, LastWrite = file.LastWrite.TotalMilliseconds, LastUpdate = UnixTime.Now.TotalMilliseconds, file.LocalSize, file.RemoteSize, Type = file.Type.ToString(), file.Hidden, file.ReadOnly, file.Deleted, file.CheckSum }) > 0; } public async Task GetLocalSizeAsync() @@ -125,11 +108,11 @@ public async Task> GetFilesAsync(string path, bool delet #region History /// - public async Task AddHistoryAsync(string path, HistoryType type) + public async Task AddHistoryAsync(string path, HistoryType type) { using IDbConnection connection = CreateConnection(); string sql = @"INSERT OR REPLACE INTO history (timestamp, name, path, type) VALUES(@Timestamp, @Name, @Path, @Type);"; - return await connection.ExecuteAsync(sql, new { Timestamp = UnixTime.Now.TotalMilliseconds, Name = Path.GetFileName(path), Path = path, Type = type }); + return await connection.ExecuteAsync(sql, new { Timestamp = UnixTime.Now.TotalMilliseconds, Name = Path.GetFileName(path), Path = path, Type = type }) > 0; } /// @@ -146,14 +129,22 @@ public async Task AddHistoryAsync(string path, HistoryType type) #endregion - #region Manifests + #region Objects + + /// + public async Task AddObjectAsync(string id, string hash, int index) + { + using IDbConnection connection = CreateConnection(); + string sql = "INSERT OR REPLACE INTO objects (id, hash, orderIndex) VALUES (@id, @hash, @index);"; + return await connection.ExecuteAsync(sql, new { id, hash, index }) > 0; + } /// - public async Task AddManifestAsync(SystemFile file) + public async Task> GetObjectsAsync(string id) { using IDbConnection connection = CreateConnection(); - string sql = @"INSERT OR REPLACE INTO manifests (name, path, lastwrite) VALUES (@Name, @Path, @LastWrite);"; - return await connection.ExecuteAsync(sql, new { file.Name, Path = file.LocalPath, LastWrite = file.LastWrite.TotalMilliseconds }); + string sql = "SELECT (hash) FROM objects WHERE id = @id ORDER BY orderIndex ASC;"; + return await connection.QueryAsync(sql, new { id }); } #endregion diff --git a/Parallel.Core/Database/IDatabase.cs b/Parallel.Core/Database/IDatabase.cs index 7444709..fddf31b 100644 --- a/Parallel.Core/Database/IDatabase.cs +++ b/Parallel.Core/Database/IDatabase.cs @@ -3,6 +3,7 @@ using Parallel.Core.IO; using System; using System.Data; +using Parallel.Core.IO.Blobs; using Parallel.Core.IO.FileSystem; using Parallel.Core.Models; @@ -47,7 +48,7 @@ public enum HistoryType /// /// An interface for interacting with client data storage. /// - public interface IDatabase : IDisposable + public interface IDatabase { #region Base @@ -72,7 +73,11 @@ public interface IDatabase : IDisposable /// /// /// True if successful, false otherwise - Task AddFileAsync(SystemFile file); + Task AddFileAsync(SystemFile file); + + Task> GetFilesAsync(string path); + Task> GetFilesAsync(string path, bool deleted); + Task GetFileAsync(string path); Task GetLocalSizeAsync(); Task GetRemoteSizeAsync(); @@ -88,7 +93,7 @@ public interface IDatabase : IDisposable /// /// /// True if successful, false otherwise - Task AddHistoryAsync(string path, HistoryType type); + Task AddHistoryAsync(string path, HistoryType type); IEnumerable? GetHistory(string path, int limit); @@ -96,10 +101,12 @@ public interface IDatabase : IDisposable #endregion - Task> GetFilesAsync(string path); - Task> GetFilesAsync(string path, bool deleted); - Task GetFileAsync(string path); - Task AddManifestAsync(SystemFile file); - Task AddChunkAsync(int manifestId, string hash, int length); + #region Objects + + Task AddObjectAsync(string id, string hash, int index); + Task> GetObjectsAsync(string id); + + #endregion + } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/BlobSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs similarity index 76% rename from Parallel.Core/IO/Syncing/BlobSyncManager.cs rename to Parallel.Core/IO/Syncing/ObjectSyncManager.cs index b549da6..e33411d 100644 --- a/Parallel.Core/IO/Syncing/BlobSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -14,7 +14,7 @@ namespace Parallel.Core.IO.Syncing /// /// Represents the way to sync files with content assigned binary objects. /// - public class BlobSyncManager : BaseSyncManager + public class ObjectSyncManager : BaseSyncManager { /// /// The size, in bytes, to use for chunks of a file. @@ -23,10 +23,10 @@ public class BlobSyncManager : BaseSyncManager /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// - public BlobSyncManager(LocalVaultConfig localVault) : base(localVault) { } + public ObjectSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// public override async Task PushFilesAsync(SystemFile[] files, IProgressReporter progress) @@ -35,7 +35,9 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options { if (file.Deleted) { - + progress.Report(ProgressOperation.Archiving, file); + await Database.AddHistoryAsync(file.LocalPath, HistoryType.Archived); + await Database.AddFileAsync(file); } else { @@ -45,12 +47,13 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options int index = 0; progress.Report(ProgressOperation.Uploading, file); - int row = await Database.AddManifestAsync(file); + await Database.AddFileAsync(file); + while ((bytesRead = await fs.ReadAsync(buffer, ct)) > 0) { await using MemoryStream ms = new MemoryStream(buffer, 0, bytesRead); string hash = HashGenerator.CreateSHA256(buffer.AsSpan(0, bytesRead)); - await Database.AddChunkAsync(row, hash, index); + await Database.AddObjectAsync(file.Id, hash, index); index++; string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); @@ -58,9 +61,14 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string remotePath = PathBuilder.Combine(parentDir, hash[4..]); if (!await FileSystem.ExistsAsync(remotePath)) { + Log.Debug($"Uploading object: {hash}"); if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); await FileSystem.UploadStreamAsync(ms, remotePath); } + else + { + Log.Debug($"Skipping object: {hash}"); + } } } }); diff --git a/Parallel.Core/IO/Syncing/SyncManager.cs b/Parallel.Core/IO/Syncing/SyncManager.cs index 9d8b84a..842fe2d 100644 --- a/Parallel.Core/IO/Syncing/SyncManager.cs +++ b/Parallel.Core/IO/Syncing/SyncManager.cs @@ -16,7 +16,7 @@ public static class SyncManager /// public static ISyncManager CreateNew(LocalVaultConfig localVault) { - return new BlobSyncManager(localVault); + return new ObjectSyncManager(localVault); } } } \ No newline at end of file diff --git a/Parallel.Core/Models/SystemFile.cs b/Parallel.Core/Models/SystemFile.cs index 65cf1bd..6410b0e 100644 --- a/Parallel.Core/Models/SystemFile.cs +++ b/Parallel.Core/Models/SystemFile.cs @@ -56,17 +56,17 @@ public class SystemFile /// /// The category of the file. /// - public FileCategory Type { get; set; } = FileCategory.Other; + public FileCategory Type { get; } = FileCategory.Other; /// /// If the file is currently hidden on the local machine. /// - public bool Hidden { get; set; } = false; + public bool Hidden { get; } = false; /// /// If the file is currently read-only on the local machine. /// - public bool ReadOnly { get; set; } = false; + public bool ReadOnly { get; } = false; /// /// If the file is currently deleted on the local machine. @@ -76,7 +76,7 @@ public class SystemFile /// /// The checksum used to check if the file has changed. /// - public byte[]? CheckSum { get; set; } + public byte[]? CheckSum { get; } /// @@ -97,6 +97,8 @@ public SystemFile(string path) ReadOnly = fileInfo.Attributes.HasFlag(FileAttributes.ReadOnly); Deleted = !fileInfo.Exists; CheckSum = HashGenerator.CheckSum(path); + // Salt = HashGenerator.RandomBytes(32); + // IV = HashGenerator.RandomBytes(32); } public SystemFile(string localPath, string remotePath) From f15a0557eacc7e4d871f783a334066d7607166f7 Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Mon, 8 Dec 2025 21:21:38 -0600 Subject: [PATCH 5/6] Can now pull files --- Parallel.Cli/Commands/PullCommand.cs | 12 +++++++++--- .../Database/Contexts/SqliteContext.cs | 9 +++++---- Parallel.Core/IO/Syncing/BaseSyncManager.cs | 6 +++--- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/Parallel.Cli/Commands/PullCommand.cs b/Parallel.Cli/Commands/PullCommand.cs index fe71931..00bfd8b 100644 --- a/Parallel.Cli/Commands/PullCommand.cs +++ b/Parallel.Cli/Commands/PullCommand.cs @@ -1,6 +1,7 @@ // Copyright 2025 Kyle Ebbinga using System.CommandLine; +using Newtonsoft.Json.Linq; using Parallel.Cli.Utils; using Parallel.Core.Diagnostics; using Parallel.Core.IO; @@ -24,10 +25,11 @@ public PullCommand() : base("pull", "Pulls changes from a vault.") this.AddOption(_forceOpt); this.SetHandler(async (path, config, force) => { - LocalVaultConfig? vault = ParallelConfig.GetVault(config); + LocalVaultConfig? vault = ParallelConfig.Load().Vaults.FirstOrDefault(); + if (!string.IsNullOrEmpty(config)) vault = ParallelConfig.GetVault(config); if (vault == null) { - CommandLine.WriteLine($"Unable to find vault with name: '{vault}'", ConsoleColor.Yellow); + CommandLine.WriteLine($"No vault was found!", ConsoleColor.Yellow); return; } @@ -54,7 +56,8 @@ private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force IEnumerable files = await syncManager.Database.GetFilesAsync(fullPath); if (!files.Any()) { - CommandLine.WriteLine("The provided directory has not been pushed!", ConsoleColor.Yellow); + CommandLine.WriteLine(vault, "No files were found!", ConsoleColor.Yellow); + await syncManager.DisconnectAsync(); return; } @@ -75,6 +78,7 @@ private async Task PullFileAsync(ISyncManager syncManager, string fullPath, bool if (remoteFile == null) { CommandLine.WriteLine("The provided file has not been pushed!", ConsoleColor.Yellow); + await syncManager.DisconnectAsync(); return; } @@ -87,6 +91,8 @@ private async Task PullFileAsync(ISyncManager syncManager, string fullPath, bool Log.Debug($"Pulling '{fullPath}'"); await syncManager.PullFilesAsync([remoteFile], new ProgressLogger()); + await syncManager.DisconnectAsync(); + CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pulled file from '{syncManager.RemoteVault.FileSystem.RootDirectory}'.", ConsoleColor.Green); } } diff --git a/Parallel.Core/Database/Contexts/SqliteContext.cs b/Parallel.Core/Database/Contexts/SqliteContext.cs index 4c24659..1ab16e8 100644 --- a/Parallel.Core/Database/Contexts/SqliteContext.cs +++ b/Parallel.Core/Database/Contexts/SqliteContext.cs @@ -4,6 +4,7 @@ using System.Data; using System.Diagnostics; using Dapper; +using Newtonsoft.Json.Linq; using Parallel.Core.IO; using Parallel.Core.IO.Blobs; using Parallel.Core.Models; @@ -83,8 +84,8 @@ public async Task GetTotalFilesAsync(bool deleted) public async Task> GetFilesAsync(string path) { using IDbConnection connection = CreateConnection(); - string sql = $"SELECT * FROM files WHERE localpath LIKE \"%@path%\" OR remotepath LIKE \"%@path%\" ORDER BY lastupdate DESC"; - return await connection.QueryAsync(sql, new { path }); + string sql = "SELECT * FROM files WHERE localpath LIKE @Path OR remotepath LIKE @Path ORDER BY lastupdate DESC"; + return await connection.QueryAsync(sql, new { Path = $"%{path}%" }); } /// @@ -99,8 +100,8 @@ public async Task> GetFilesAsync(string path, bool delet public async Task GetFileAsync(string path) { using IDbConnection connection = CreateConnection(); - string sql = $"SELECT (id, name, localpath, remotepath, lastwrite, lastupdate, LocalSize, RemoteSize, type, hidden, readonly, deleted, checksum) FROM files WHERE localpath LIKE \"%{path}%\" OR remotepath LIKE \"%{path}%\" ORDER BY lastupdate DESC"; - return await connection.QuerySingleOrDefaultAsync(sql); + string sql = $"SELECT (id, name, localpath, remotepath, lastwrite, lastupdate, localsize, remotesize, type, hidden, readonly, deleted, checksum) FROM files WHERE localpath LIKE \"%@path%\" OR remotepath LIKE \"%@path%\" ORDER BY lastupdate DESC"; + return await connection.QuerySingleOrDefaultAsync(sql, new { path }); } #endregion diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index 8a54ba8..4280bdf 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -59,7 +59,7 @@ public async Task ConnectAsync() } else { - await FileSystem.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new ProgressLogger()); + await FileSystem.DownloadFilesAsync([new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault))], new NullProgressReporter()); RemoteVaultConfig? config = RemoteVaultConfig.Load(TempConfigFile); if(config == null) return false; RemoteVault = config; @@ -76,7 +76,7 @@ public async Task ConnectAsync() } else { - await FileSystem.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new ProgressLogger()); + await FileSystem.DownloadFilesAsync([new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))], new NullProgressReporter()); Database = new SqliteContext(TempDbFile); Log.Debug($"Downloaded db file: {TempDbFile}"); @@ -92,7 +92,7 @@ public async Task DisconnectAsync() Log.Debug($"Uploaded db file: {TempDbFile}"); SystemFile[] tempFiles = [new SystemFile(TempConfigFile, PathBuilder.GetConfigurationFile(LocalVault)), new SystemFile(TempDbFile, PathBuilder.GetDatabaseFile(LocalVault))]; - await FileSystem.UploadFilesAsync(tempFiles, new ProgressLogger()); + await FileSystem.UploadFilesAsync(tempFiles, new NullProgressReporter()); FileSystem.Dispose(); } diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index e33411d..c4965a6 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -77,6 +77,24 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options /// public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter progress) { + await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + { + progress.Report(ProgressOperation.Downloading, file); + await using FileStream fs = File.Create(file.LocalPath); + foreach (string hash in await Database.GetObjectsAsync(file.Id)) + { + string basePath = PathBuilder.Combine(RemoteVault.FileSystem.RootDirectory, "Parallel", RemoteVault.Id, "objects"); + string remotePath = PathBuilder.Combine(basePath, hash.Substring(0, 2), hash.Substring(2, 2), hash[4..]); + + if (await FileSystem.ExistsAsync(remotePath)) + { + Log.Debug($"Downloading object: {hash}"); + await FileSystem.DownloadStreamAsync(fs, remotePath); + } + } + + await fs.FlushAsync(ct); + }); } } } \ No newline at end of file From 8482d7ff54f341536fed2def7e943fba6630a6ad Mon Sep 17 00:00:00 2001 From: Kyle | Guitar <26614720+TheGuitarleader@users.noreply.github.com> Date: Tue, 9 Dec 2025 02:28:20 -0600 Subject: [PATCH 6/6] Various minor improvements --- Parallel.Cli/Commands/DiskCommand.cs | 17 ++-- Parallel.Cli/Commands/PullCommand.cs | 5 +- Parallel.Cli/Commands/PushCommand.cs | 3 + Parallel.Cli/Commands/RemapCommand.cs | 13 +++ Parallel.Cli/Utils/TextWriter.cs | 2 +- Parallel.Core/IO/Scanning/FileScanner.cs | 96 ++++++++++--------- Parallel.Core/IO/Syncing/ObjectSyncManager.cs | 8 +- 7 files changed, 85 insertions(+), 59 deletions(-) create mode 100644 Parallel.Cli/Commands/RemapCommand.cs diff --git a/Parallel.Cli/Commands/DiskCommand.cs b/Parallel.Cli/Commands/DiskCommand.cs index df4e46f..4753ae6 100644 --- a/Parallel.Cli/Commands/DiskCommand.cs +++ b/Parallel.Cli/Commands/DiskCommand.cs @@ -14,27 +14,28 @@ namespace Parallel.Cli.Commands { public class DiskCommand : Command { - private readonly Argument configArg = new("config", "The vault configuration to use."); + private readonly Option _configOpt = new(["--config", "-c"], "The vault configuration to use."); public DiskCommand() : base("disk", "Shows the current disk usage.") { - this.AddArgument(configArg); - this.SetHandler(async (vault) => + this.AddOption(_configOpt); + this.SetHandler(async (config) => { - CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); - LocalVaultConfig? config = ParallelConfig.GetVault(vault); - if (config == null) + LocalVaultConfig? vault = ParallelConfig.Load().Vaults.FirstOrDefault(); + if (!string.IsNullOrEmpty(config)) vault = ParallelConfig.GetVault(config); + if (vault == null) { CommandLine.WriteLine($"Unable to find vault with name: '{vault}'", ConsoleColor.Yellow); return; } - await DisplayDiskInformationAsync(config); - }, configArg); + await DisplayDiskInformationAsync(vault); + }, _configOpt); } private async Task DisplayDiskInformationAsync(LocalVaultConfig vault) { + CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); ISyncManager syncManager = SyncManager.CreateNew(vault); if (!await syncManager.ConnectAsync()) { diff --git a/Parallel.Cli/Commands/PullCommand.cs b/Parallel.Cli/Commands/PullCommand.cs index 00bfd8b..0b2c895 100644 --- a/Parallel.Cli/Commands/PullCommand.cs +++ b/Parallel.Cli/Commands/PullCommand.cs @@ -39,6 +39,7 @@ public PullCommand() : base("pull", "Pulls changes from a vault.") private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force) { + CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); ISyncManager syncManager = SyncManager.CreateNew(vault); if (!await syncManager.ConnectAsync()) { @@ -62,7 +63,7 @@ private async Task PullPathAsync(LocalVaultConfig vault, string path, bool force } List pullFiles = new List(); - await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => + System.Threading.Tasks.Parallel.ForEach(files, ParallelConfig.Options, (file) => { if (!File.Exists(file.LocalPath) || FileScanner.HasChanged(file, new SystemFile(file.LocalPath)) || force) pullFiles.Add(file); }); @@ -77,7 +78,7 @@ private async Task PullFileAsync(ISyncManager syncManager, string fullPath, bool SystemFile? remoteFile = await syncManager.Database.GetFileAsync(fullPath); if (remoteFile == null) { - CommandLine.WriteLine("The provided file has not been pushed!", ConsoleColor.Yellow); + CommandLine.WriteLine("The provided file was not found!", ConsoleColor.Yellow); await syncManager.DisconnectAsync(); return; } diff --git a/Parallel.Cli/Commands/PushCommand.cs b/Parallel.Cli/Commands/PushCommand.cs index f5d388d..6668fcb 100644 --- a/Parallel.Cli/Commands/PushCommand.cs +++ b/Parallel.Cli/Commands/PushCommand.cs @@ -47,6 +47,7 @@ private Task SyncSystemAsync() private async Task SyncPathAsync(string path) { + CommandLine.WriteLine($"Retrieving vault information...", ConsoleColor.DarkGray); await Program.Settings.ForEachVaultAsync(async vault => { ISyncManager syncManager = SyncManager.CreateNew(vault); @@ -65,12 +66,14 @@ await Program.Settings.ForEachVaultAsync(async vault => if (!backupFolders.Any(dir => fullPath.StartsWith(dir, StringComparison.OrdinalIgnoreCase))) { CommandLine.WriteLine(vault, $"The provided {(isFile ? "file" : "folder")} is not set to be backed up!", ConsoleColor.Yellow); + await syncManager.DisconnectAsync(); return; } if (FileScanner.IsIgnored(fullPath, ignoredFolders)) { CommandLine.WriteLine(vault, $"The provided {(isFile ? "file" : "folder")} is set to be ignored!", ConsoleColor.Yellow); + await syncManager.DisconnectAsync(); return; } diff --git a/Parallel.Cli/Commands/RemapCommand.cs b/Parallel.Cli/Commands/RemapCommand.cs new file mode 100644 index 0000000..459193d --- /dev/null +++ b/Parallel.Cli/Commands/RemapCommand.cs @@ -0,0 +1,13 @@ +// Copyright 2025 Kyle Ebbinga + +using System.CommandLine; + +namespace Parallel.Cli.Commands +{ + public class RemapCommand : Command + { + public RemapCommand() : base("remap", "Remaps paths in the vault.") + { + } + } +} \ No newline at end of file diff --git a/Parallel.Cli/Utils/TextWriter.cs b/Parallel.Cli/Utils/TextWriter.cs index 01848bc..08d4587 100644 --- a/Parallel.Cli/Utils/TextWriter.cs +++ b/Parallel.Cli/Utils/TextWriter.cs @@ -5,7 +5,7 @@ namespace Parallel.Cli.Utils { - public class TextWriter + public abstract class TextWriter { public static string CreateTxtFile(string text) { diff --git a/Parallel.Core/IO/Scanning/FileScanner.cs b/Parallel.Core/IO/Scanning/FileScanner.cs index 8b09b42..cdd0ac2 100644 --- a/Parallel.Core/IO/Scanning/FileScanner.cs +++ b/Parallel.Core/IO/Scanning/FileScanner.cs @@ -4,6 +4,7 @@ using System.Data; using System.Diagnostics; using System.Text; +using System.Text.RegularExpressions; using Newtonsoft.Json.Linq; using Parallel.Core.Database; using Parallel.Core.IO.Syncing; @@ -19,6 +20,7 @@ namespace Parallel.Core.IO.Scanning /// public class FileScanner { + private static readonly Dictionary _cache = new(); private readonly RemoteVaultConfig _config; private readonly IDatabase _db; @@ -95,11 +97,8 @@ public async Task GetFileChangesAsync(string path, string[] ignore { if (!IsIgnored(file, ignoreFolders)) { - if (File.Exists(file)) - { - //Log.Debug($"Created -> {file}"); - changedFiles.Add(new SystemFile(file) { RemotePath = PathBuilder.Remote(file, _config) }); - } + //Log.Debug($"Created -> {file}"); + changedFiles.Add(new SystemFile(file) { RemotePath = PathBuilder.Remote(file, _config) }); } }); @@ -235,31 +234,31 @@ public static IEnumerable GetFiles(string root, string[] exempt, string } // Get files in current directory - string[] files = []; + IEnumerable files; try { - files = Directory.GetFiles(current, searchPattern); + files = Directory.EnumerateFiles(current, searchPattern); } catch { Log.Debug($"No file access -> {current}"); + continue; } - foreach (var file in files) yield return file; - if (recursive) + foreach (string file in files) yield return file; + if (!recursive) continue; + IEnumerable subDirs; + try { - string[] subdirs = []; - try - { - subdirs = Directory.GetDirectories(current); - } - catch - { - Log.Debug($"No folder access -> {current}"); - } - - foreach (var dir in subdirs) pending.Push(dir); + subDirs = Directory.EnumerateDirectories(current); } + catch + { + Log.Debug($"No directory access -> {current}"); + continue; + } + + foreach (string dir in subDirs) pending.Push(dir); } } @@ -278,8 +277,8 @@ public static Dictionary GetDuplicateFiles(string path) SystemFile entry = new(file); if (dict.TryGetValue(entry.Name, out List value)) { - SystemFile key = value.FirstOrDefault(); - if (entry.LocalSize.Equals(key.LocalSize)) + SystemFile? key = value.FirstOrDefault(); + if (entry.LocalSize.Equals(key?.LocalSize)) { value.Add(entry); } @@ -323,33 +322,44 @@ public static bool IsIgnored(string path, string[] exempt) { foreach (string entry in exempt) { - if (path.StartsWith(entry)) + if (!_cache.TryGetValue(entry, out Regex? regex)) { - return true; + regex = BuildRegexCache(entry); + _cache[entry] = regex; } - if (entry.EndsWith('/')) - { - string[] folders = path.Split('\\'); - foreach (string dir in folders) - { - if (dir.ToLower() == entry.Remove(entry.Length - 1, 1).ToLower()) - { - return true; - } - } - } - - if (entry.StartsWith('*')) - { - if (path.EndsWith(entry.Replace("*", string.Empty))) - { - return true; - } - } + if (regex.IsMatch(path)) + return true; } return false; } + + private static Regex BuildRegexCache(string entry) + { + string pattern; + if (!entry.Contains('*') && !entry.EndsWith("/") && !entry.EndsWith("\\")) + { + pattern = "^" + Regex.Escape(entry); + return new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); + } + + if (entry.EndsWith("/") || entry.EndsWith("\\")) + { + string folder = entry.TrimEnd('/', '\\'); + pattern = @"(^|\\)" + Regex.Escape(folder) + @"(\\|$)"; + return new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); + } + + if (entry.StartsWith("*")) + { + string ext = Regex.Escape(entry.TrimStart('*')); + pattern = ".*" + ext + "$"; + return new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); + } + + pattern = "^" + Regex.Escape(entry) + "$"; + return new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); + } } } \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs index c4965a6..3c6f417 100644 --- a/Parallel.Core/IO/Syncing/ObjectSyncManager.cs +++ b/Parallel.Core/IO/Syncing/ObjectSyncManager.cs @@ -61,14 +61,9 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options string remotePath = PathBuilder.Combine(parentDir, hash[4..]); if (!await FileSystem.ExistsAsync(remotePath)) { - Log.Debug($"Uploading object: {hash}"); if (!await FileSystem.ExistsAsync(parentDir)) await FileSystem.CreateDirectoryAsync(parentDir); await FileSystem.UploadStreamAsync(ms, remotePath); } - else - { - Log.Debug($"Skipping object: {hash}"); - } } } }); @@ -80,6 +75,9 @@ public override async Task PullFilesAsync(SystemFile[] files, IProgressReporter await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { progress.Report(ProgressOperation.Downloading, file); + string? parentDir = Path.GetDirectoryName(file.LocalPath); + if (!Directory.Exists(parentDir)) Directory.CreateDirectory(parentDir); + await using FileStream fs = File.Create(file.LocalPath); foreach (string hash in await Database.GetObjectsAsync(file.Id)) {