From 5af01b3deaeff018b1318997d471a04f1840b541 Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Fri, 16 Oct 2026 00:48:12 +0900 Subject: [PATCH 1/4] Give AnalysisFileBean a raw data format (key 19) and a classifier that tells it without reading the file mzTab-M names each run's file format by looking at the disk at export time, so once the raw data are deleted a re-export writes "Unknown .raw format" or "Unknown .d folder format" for a format MS-DIAL knew when it processed the file. This adds the place to keep what was known. - RawFileFormat (MsdialCore/Enum): one value per format the exporter tells apart, persisted by number. Numbers are explicit and append-only; Unknown is 0. - AnalysisFileBean.RawFileFormat as MessagePack key 19, without an initializer, so a project written before it reads as Unknown. Key 19 is unused on master, on every remote branch and on the 5.5/5.6 tags. - RawFileFormatClassifier (MsdialCore/Utility, no UI dependency): - Classify(path) tells the format by extension, in any letter case. A .raw is a Waters folder or a Thermo file. A .d is checked in RawDataHandler's order, AcqData first, then analysis.tdf, .tsf and .baf. It never throws: a path that cannot be checked is Unknown. - RecordIfUnknown(file) fills the field only when nothing is recorded. - IsConsistentWith(format, extension) and Resolve(file): the recorded format when it fits the path, otherwise what the disk tells now. Tests pin every extension and its letter cases, the .raw and .d shapes (including a .d holding both AcqData and a Bruker file), unusable paths, the enum numbers, a round trip of every value, that the 2025-07-31 fixture project (written with 19 keys) reads as Unknown with keys 0-18 unchanged, and that a reader declaring only keys 0-18 reads a bean with key 19, alone and inside a list followed by more data. Co-Authored-By: Claude Opus 5.5 --- .../MsdialCore/DataObj/AnalysisFile.cs | 11 +- src/MSDIAL5/MsdialCore/Enum/RawFileFormat.cs | 34 +++ .../Utility/RawFileFormatClassifier.cs | 156 +++++++++++ .../AnalysisFileBeanRawFileFormatTests.cs | 258 +++++++++++++++++ .../Utility/RawFileFormatClassifierTests.cs | 261 ++++++++++++++++++ 5 files changed, 719 insertions(+), 1 deletion(-) create mode 100644 src/MSDIAL5/MsdialCore/Enum/RawFileFormat.cs create mode 100644 src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs create mode 100644 tests/MSDIAL5/MsdialCoreTests/DataObj/AnalysisFileBeanRawFileFormatTests.cs create mode 100644 tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs diff --git a/src/MSDIAL5/MsdialCore/DataObj/AnalysisFile.cs b/src/MSDIAL5/MsdialCore/DataObj/AnalysisFile.cs index e224e37e4..d64468ab9 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/AnalysisFile.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/AnalysisFile.cs @@ -1,6 +1,7 @@ using CompMs.Common.DataObj; using CompMs.Common.Enum; using CompMs.MsdialCore.Algorithm; +using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.MSDec; using CompMs.MsdialCore.Parameter; using CompMs.MsdialCore.Parser; @@ -62,7 +63,15 @@ public AnalysisFileBean() { public string ProteinAssembledResultFilePath { get; set; } // *.prf [Key(18)] public AcquisitionType AcquisitionType { get; set; } = AcquisitionType.None; - + /// + /// The format of the raw data file, recorded while the file was present (see ), + /// so that it can still be named after the raw data are deleted or moved. when + /// it was never seen, which is also what a project written before this member existed reads as. + /// + /// No initializer: a key missing from an older project is read as default(T), not as an initializer's value. + [Key(19)] + public RawFileFormat RawFileFormat { get; set; } + [IgnoreMember] public bool IsDoMs2ChromDeconvolution => AcquisitionType != AcquisitionType.DDA; diff --git a/src/MSDIAL5/MsdialCore/Enum/RawFileFormat.cs b/src/MSDIAL5/MsdialCore/Enum/RawFileFormat.cs new file mode 100644 index 000000000..ce67e91e6 --- /dev/null +++ b/src/MSDIAL5/MsdialCore/Enum/RawFileFormat.cs @@ -0,0 +1,34 @@ +namespace CompMs.MsdialCore.Enum { + /// + /// What a raw data file is, as far as MS-DIAL can tell from its extension and, for the vendor folders + /// and the two kinds of .raw, from what is on disk. Recorded on + /// when the file is registered, so the format can still be named after the raw data are gone. + /// + /// + /// Persisted by number in projects. Numbers are never reused or renumbered: a new format is appended + /// with the next free number, and stays 0, which is what a project written before + /// the format was recorded reads as. + /// + public enum RawFileFormat + { + Unknown = 0, + ThermoRaw = 1, + WatersRaw = 2, + AgilentMassHunterD = 3, + BrukerTdf = 4, + BrukerTsf = 5, + BrukerBaf = 6, + SciexWiff = 7, + SciexWiff2 = 8, + ShimadzuLcd = 9, + ShimadzuQgd = 10, + MzML = 11, + ImzML = 12, + NetCdf = 13, + Abf = 14, + Ibf = 15, + KanomaxLrp = 16, + HiveHmd = 17, + HiveMzb = 18, + } +} diff --git a/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs b/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs new file mode 100644 index 000000000..8bceaae74 --- /dev/null +++ b/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs @@ -0,0 +1,156 @@ +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; +using System; +using System.IO; + +namespace CompMs.MsdialCore.Utility +{ + /// + /// Tells the format of a raw data file without reading it, so that it can be recorded on + /// while the file is present. + /// + /// + /// Most formats are told by the extension alone, in any letter case. Two extensions need the disk: a .raw is a + /// Waters folder or a Thermo file, and a .d folder is Agilent's when it holds AcqData and Bruker's when it holds + /// analysis.tdf, analysis.tsf or analysis.baf. A .d is checked in the order RawDataHandler reads it, AcqData + /// first, so the format named is the one whose reader MS-DIAL used. Without the raw data on disk these two are + /// . Nothing here throws: a path that cannot be checked is Unknown. + /// + public static class RawFileFormatClassifier + { + /// + /// The format of the raw data file at , or when it + /// cannot be told. + /// + public static RawFileFormat Classify(string path) { + if (string.IsNullOrEmpty(path)) { + return RawFileFormat.Unknown; + } + try { + switch (Path.GetExtension(path).ToLowerInvariant()) { + case ".raw": return ClassifyDotRaw(path); + case ".d": return ClassifyDotD(path); + case ".wiff": return RawFileFormat.SciexWiff; + case ".wiff2": return RawFileFormat.SciexWiff2; + case ".lcd": return RawFileFormat.ShimadzuLcd; + case ".qgd": return RawFileFormat.ShimadzuQgd; + case ".mzml": return RawFileFormat.MzML; + case ".imzml": return RawFileFormat.ImzML; + case ".cdf": return RawFileFormat.NetCdf; + case ".abf": return RawFileFormat.Abf; + case ".ibf": return RawFileFormat.Ibf; + case ".lrp": return RawFileFormat.KanomaxLrp; + case ".hmd": return RawFileFormat.HiveHmd; + case ".mzb": return RawFileFormat.HiveMzb; + default: return RawFileFormat.Unknown; + } + } + catch (Exception ex) when (IsPathFailure(ex)) { + return RawFileFormat.Unknown; + } + } + + /// + /// Records the format of when none is recorded yet and its raw data tell it. + /// A recorded format is never replaced. Returns whether a format was recorded. + /// + public static bool RecordIfUnknown(AnalysisFileBean file) { + if (file is null || file.RawFileFormat != RawFileFormat.Unknown) { + return false; + } + var format = Classify(file.AnalysisFilePath); + if (format == RawFileFormat.Unknown) { + return false; + } + file.RawFileFormat = format; + return true; + } + + /// + /// Whether a file with (with its leading dot, any letter case) can be of + /// . is consistent with nothing. + /// + public static bool IsConsistentWith(RawFileFormat format, string extension) { + var expected = ExtensionOf(format); + return expected is not null && string.Equals(expected, extension, StringComparison.OrdinalIgnoreCase); + } + + /// + /// The format to describe by: the recorded one when it fits the file's extension, which + /// holds after the raw data are gone; otherwise what tells from the disk now. + /// + public static RawFileFormat Resolve(AnalysisFileBean file) { + var recorded = file.RawFileFormat; + if (recorded != RawFileFormat.Unknown && IsConsistentWith(recorded, ExtensionOrNull(file.AnalysisFilePath))) { + return recorded; + } + return Classify(file.AnalysisFilePath); + } + + private static RawFileFormat ClassifyDotRaw(string path) { + if (Directory.Exists(path)) { + return RawFileFormat.WatersRaw; + } + if (File.Exists(path)) { + return RawFileFormat.ThermoRaw; + } + return RawFileFormat.Unknown; + } + + private static RawFileFormat ClassifyDotD(string path) { + if (Directory.Exists(Path.Combine(path, "AcqData"))) { + return RawFileFormat.AgilentMassHunterD; + } + if (File.Exists(Path.Combine(path, "analysis.tdf"))) { + return RawFileFormat.BrukerTdf; + } + if (File.Exists(Path.Combine(path, "analysis.tsf"))) { + return RawFileFormat.BrukerTsf; + } + if (File.Exists(Path.Combine(path, "analysis.baf"))) { + return RawFileFormat.BrukerBaf; + } + return RawFileFormat.Unknown; + } + + private static string? ExtensionOf(RawFileFormat format) { + switch (format) { + case RawFileFormat.ThermoRaw: + case RawFileFormat.WatersRaw: + return ".raw"; + case RawFileFormat.AgilentMassHunterD: + case RawFileFormat.BrukerTdf: + case RawFileFormat.BrukerTsf: + case RawFileFormat.BrukerBaf: + return ".d"; + case RawFileFormat.SciexWiff: return ".wiff"; + case RawFileFormat.SciexWiff2: return ".wiff2"; + case RawFileFormat.ShimadzuLcd: return ".lcd"; + case RawFileFormat.ShimadzuQgd: return ".qgd"; + case RawFileFormat.MzML: return ".mzml"; + case RawFileFormat.ImzML: return ".imzml"; + case RawFileFormat.NetCdf: return ".cdf"; + case RawFileFormat.Abf: return ".abf"; + case RawFileFormat.Ibf: return ".ibf"; + case RawFileFormat.KanomaxLrp: return ".lrp"; + case RawFileFormat.HiveHmd: return ".hmd"; + case RawFileFormat.HiveMzb: return ".mzb"; + default: return null; + } + } + + private static string? ExtensionOrNull(string path) { + try { + return Path.GetExtension(path); + } + catch (Exception ex) when (IsPathFailure(ex)) { + return null; + } + } + + private static bool IsPathFailure(Exception ex) { + return ex is ArgumentException || ex is IOException || ex is NotSupportedException + || ex is UnauthorizedAccessException || ex is System.Security.SecurityException; + } + } +} diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/AnalysisFileBeanRawFileFormatTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/AnalysisFileBeanRawFileFormatTests.cs new file mode 100644 index 000000000..2f227be0a --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/AnalysisFileBeanRawFileFormatTests.cs @@ -0,0 +1,258 @@ +using CompMs.Common.Enum; +using CompMs.Common.MessagePack; +using CompMs.MsdialCore.Enum; +using MessagePack; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace CompMs.MsdialCore.DataObj.Tests +{ + /// + /// The raw data format is recorded as MessagePack key 19 of . Projects written before + /// it must still load and read as , and an MS-DIAL that knows only keys 0-18 + /// (5.5, and the 5.6 line at the time of writing) must still open a project that has it. + /// + /// + /// Project IO goes through (LZ4 over the stock resolver), where an int-keyed + /// [MessagePackObject] is an array indexed by key: a shorter array is read to its own length, and the elements past + /// the keys a reader knows are skipped. These tests pin both directions for this type, as + /// does for match results. + /// + [TestClass()] + public class AnalysisFileBeanRawFileFormatTests + { + [TestMethod()] + public void EveryFormatSurvivesSavingAndLoading() { + foreach (var format in AllFormats()) { + var bean = new AnalysisFileBean { AnalysisFileName = "sample", RawFileFormat = format, }; + + var actual = RoundTrip(bean); + + Assert.AreEqual(format, actual.RawFileFormat); + Assert.AreEqual("sample", actual.AnalysisFileName); + } + } + + [TestMethod()] + public void ANewBeanRecordsNothing() { + Assert.AreEqual(RawFileFormat.Unknown, new AnalysisFileBean().RawFileFormat); + } + + /// + /// A dataset saved on 2025-07-31, before key 19 existed: every file reads as Unknown, and keys 0-18 read exactly as + /// an MS-DIAL without key 19 reads them. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + public void AProjectWrittenBeforeTheFormatWasRecordedReadsAsUnknown() { + var bytes = File.ReadAllBytes(Path.Combine("Resources", "Export", "Dataset_2025_07_31_12_31_11.mddata")); + + var storage = LZ4MessagePackSerializer.Deserialize(bytes); + + Assert.IsTrue(storage.AnalysisFiles.Count > 0); + Assert.IsTrue(storage.AnalysisFiles.All(file => file.RawFileFormat == RawFileFormat.Unknown)); + + var beans = AnalysisFileBlocks(LZ4MessagePackSerializer.Decode(bytes)).ToList(); + Assert.AreEqual(storage.AnalysisFiles.Count, beans.Count); + foreach (var (block, actual) in beans.Zip(storage.AnalysisFiles, (block, actual) => (block, actual))) { + Assert.AreEqual(19, MessagePackBinary.ReadArrayHeader(block, 0, out _), "the fixture must predate key 19"); + var old = MessagePackSerializer.Deserialize(block); + AssertSameKeys0To18(old, actual); + } + } + + /// + /// The other direction: an MS-DIAL that knows keys 0-18 opens a project written with key 19. The beans sit in a list + /// with more data after it, so a reader that skipped key 19 by the wrong length would misread what follows. + /// + [TestMethod()] + public void AnOlderMsdialReadsAProjectWithTheFormatRecorded() { + var written = new BeansThenMore + { + AnalysisFiles = new List { CreateFullBean(0, RawFileFormat.BrukerTdf), CreateFullBean(1, RawFileFormat.HiveMzb), }, + After = "after the beans", + }; + var stream = new MemoryStream(); + MessagePackDefaultHandler.SaveToStream(written, stream); + + stream.Position = 0; + var read = MessagePackDefaultHandler.LoadFromStream(stream); + + Assert.AreEqual(2, read.AnalysisFiles.Count); + Assert.AreEqual("after the beans", read.After); + for (int i = 0; i < 2; i++) { + AssertSameKeys0To18(read.AnalysisFiles[i], written.AnalysisFiles[i]); + } + } + + [TestMethod()] + public void AnOlderMsdialReadsASingleBeanWithTheFormatRecorded() { + var bean = CreateFullBean(3, RawFileFormat.WatersRaw); + var bytes = MessagePackSerializer.Serialize(bean); + Assert.AreEqual(20, MessagePackBinary.ReadArrayHeader(bytes, 0, out _), "key 19 is written"); + + var old = MessagePackSerializer.Deserialize(bytes); + + AssertSameKeys0To18(old, bean); + } + + /// + /// Projects store the number, so a number once given to a format is never given to another. + /// New formats are appended. + /// + [TestMethod()] + public void TheFormatNumbersAreFixed() { + var expected = new Dictionary + { + [RawFileFormat.Unknown] = 0, + [RawFileFormat.ThermoRaw] = 1, + [RawFileFormat.WatersRaw] = 2, + [RawFileFormat.AgilentMassHunterD] = 3, + [RawFileFormat.BrukerTdf] = 4, + [RawFileFormat.BrukerTsf] = 5, + [RawFileFormat.BrukerBaf] = 6, + [RawFileFormat.SciexWiff] = 7, + [RawFileFormat.SciexWiff2] = 8, + [RawFileFormat.ShimadzuLcd] = 9, + [RawFileFormat.ShimadzuQgd] = 10, + [RawFileFormat.MzML] = 11, + [RawFileFormat.ImzML] = 12, + [RawFileFormat.NetCdf] = 13, + [RawFileFormat.Abf] = 14, + [RawFileFormat.Ibf] = 15, + [RawFileFormat.KanomaxLrp] = 16, + [RawFileFormat.HiveHmd] = 17, + [RawFileFormat.HiveMzb] = 18, + }; + foreach (var pair in expected) { + Assert.AreEqual(pair.Value, (int)pair.Key, pair.Key.ToString()); + } + // A value added at the end needs a line above; one added anywhere else breaks a line above. + CollectionAssert.AreEquivalent(expected.Keys.ToArray(), AllFormats().ToArray()); + } + + private static IEnumerable AllFormats() { + return System.Enum.GetValues(typeof(RawFileFormat)).Cast(); + } + + private static AnalysisFileBean RoundTrip(AnalysisFileBean bean) { + using var stream = new MemoryStream(); + MessagePackDefaultHandler.SaveToStream(bean, stream); + stream.Position = 0; + return MessagePackDefaultHandler.LoadFromStream(stream); + } + + /// + /// The MessagePack block of each analysis file in a serialized , whose key 0 is the + /// list of analysis files. + /// + private static IEnumerable AnalysisFileBlocks(byte[] storage) { + var offset = 0; + MessagePackBinary.ReadArrayHeader(storage, offset, out var readSize); + offset += readSize; + var count = MessagePackBinary.ReadArrayHeader(storage, offset, out readSize); + offset += readSize; + for (int i = 0; i < count; i++) { + var size = MessagePackBinary.ReadNextBlock(storage, offset); + var block = new byte[size]; + Array.Copy(storage, offset, block, 0, size); + offset += size; + yield return block; + } + } + + private static AnalysisFileBean CreateFullBean(int id, RawFileFormat format) { + return new AnalysisFileBean + { + AnalysisFilePath = $@"D:\data\sample_{id}.d", + AnalysisFileName = $"sample_{id}", + AnalysisFileType = AnalysisFileType.QC, + AnalysisFileClass = "class A", + AnalysisFileAnalyticalOrder = 10 + id, + AnalysisFileId = id, + AnalysisFileIncluded = true, + DeconvolutionFilePath = $@"D:\data\sample_{id}.dcl", + DeconvolutionFilePathList = new List { $@"D:\data\sample_{id}_1000.dcl", $@"D:\data\sample_{id}_2000.dcl", }, + PeakAreaBeanInformationFilePath = $@"D:\data\sample_{id}.pai", + RiDictionaryFilePath = $@"D:\data\ri_{id}.txt", + AnalysisBatch = 2, + ResponseVariable = 0.25, + DilutionFactor = 4d, + AnalysisFileSuperClass = "super", + RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean($@"D:\data\sample_{id}.rtc"), + ProteinAssembledResultFilePath = $@"D:\data\sample_{id}.prf", + AcquisitionType = AcquisitionType.SWATH, + RawFileFormat = format, + }; + } + + /// + /// Keys 0-18 must be read the same by both. Each key is compared by its MessagePack encoding, so nested objects + /// (keys 15 and 16) are compared whole. + /// + private static void AssertSameKeys0To18(AnalysisFileBeanWithoutRawFileFormat old, AnalysisFileBean actual) { + var oldBytes = MessagePackSerializer.Serialize(old); + var actualBytes = MessagePackSerializer.Serialize(actual); + Assert.AreEqual(19, MessagePackBinary.ReadArrayHeader(oldBytes, 0, out var oldOffset)); + Assert.AreEqual(20, MessagePackBinary.ReadArrayHeader(actualBytes, 0, out var actualOffset)); + for (int key = 0; key < 19; key++) { + var oldSize = MessagePackBinary.ReadNextBlock(oldBytes, oldOffset); + var actualSize = MessagePackBinary.ReadNextBlock(actualBytes, actualOffset); + CollectionAssert.AreEqual( + new ArraySegment(oldBytes, oldOffset, oldSize).ToArray(), + new ArraySegment(actualBytes, actualOffset, actualSize).ToArray(), + $"key {key}"); + oldOffset += oldSize; + actualOffset += actualSize; + } + Assert.AreEqual(old.AnalysisFilePath, actual.AnalysisFilePath); + Assert.AreEqual(old.AnalysisFileName, actual.AnalysisFileName); + Assert.AreEqual(old.AnalysisFileId, actual.AnalysisFileId); + Assert.AreEqual(old.AcquisitionType, actual.AcquisitionType); + } + + /// + /// as an MS-DIAL without key 19 declares it. + /// + [MessagePackObject] + public class AnalysisFileBeanWithoutRawFileFormat + { + [Key(0)] public string AnalysisFilePath { get; set; } = string.Empty; + [Key(1)] public string AnalysisFileName { get; set; } = string.Empty; + [Key(2)] public AnalysisFileType AnalysisFileType { get; set; } + [Key(3)] public string AnalysisFileClass { get; set; } = string.Empty; + [Key(4)] public int AnalysisFileAnalyticalOrder { get; set; } + [Key(5)] public int AnalysisFileId { get; set; } + [Key(6)] public bool AnalysisFileIncluded { get; set; } + [Key(7)] public string DeconvolutionFilePath { get; set; } = string.Empty; + [Key(8)] public List DeconvolutionFilePathList { get; set; } = new List(); + [Key(9)] public string PeakAreaBeanInformationFilePath { get; set; } = string.Empty; + [Key(10)] public string RiDictionaryFilePath { get; set; } = string.Empty; + [Key(11)] public int AnalysisBatch { get; set; } = 1; + [Key(12)] public double ResponseVariable { get; set; } = 0; + [Key(13)] public double DilutionFactor { get; set; } = 1.0; + [Key(14)] public string AnalysisFileSuperClass { get; set; } = string.Empty; + [Key(15)] public RetentionTimeCorrectionBean RetentionTimeCorrectionBean { get; set; } = new RetentionTimeCorrectionBean(); + [Key(16)] public ChromatogramPeaksDataSummaryDto ChromPeakFeaturesSummary { get; set; } = new ChromatogramPeaksDataSummaryDto(); + [Key(17)] public string ProteinAssembledResultFilePath { get; set; } + [Key(18)] public AcquisitionType AcquisitionType { get; set; } = AcquisitionType.None; + } + + [MessagePackObject] + public class BeansThenMore + { + [Key(0)] public List AnalysisFiles { get; set; } = new List(); + [Key(1)] public string After { get; set; } = string.Empty; + } + + [MessagePackObject] + public class OldBeansThenMore + { + [Key(0)] public List AnalysisFiles { get; set; } = new List(); + [Key(1)] public string After { get; set; } = string.Empty; + } + } +} diff --git a/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs b/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs new file mode 100644 index 000000000..6e8b0ef8e --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs @@ -0,0 +1,261 @@ +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; + +namespace CompMs.MsdialCore.Utility.Tests; + +[TestClass()] +public class RawFileFormatClassifierTests +{ + private string _folder = string.Empty; + + [TestInitialize()] + public void Setup() { + _folder = Path.Combine(Path.GetTempPath(), $"msdial-rawfileformat-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_folder); + } + + [TestCleanup()] + public void Cleanup() { + Directory.Delete(_folder, recursive: true); + } + + /// + /// These formats are told by the extension alone, in any letter case, so the file need not be there. + /// + [DataTestMethod()] + [DataRow("sample.abf", RawFileFormat.Abf)] + [DataRow("sample.ABF", RawFileFormat.Abf)] + [DataRow("sample.ibf", RawFileFormat.Ibf)] + [DataRow("sample.IBF", RawFileFormat.Ibf)] + [DataRow("sample.wiff", RawFileFormat.SciexWiff)] + [DataRow("sample.WIFF", RawFileFormat.SciexWiff)] + [DataRow("sample.wiff2", RawFileFormat.SciexWiff2)] + [DataRow("sample.WIFF2", RawFileFormat.SciexWiff2)] + [DataRow("sample.lcd", RawFileFormat.ShimadzuLcd)] + [DataRow("sample.LCD", RawFileFormat.ShimadzuLcd)] + [DataRow("sample.qgd", RawFileFormat.ShimadzuQgd)] + [DataRow("sample.QGD", RawFileFormat.ShimadzuQgd)] + [DataRow("sample.mzML", RawFileFormat.MzML)] + [DataRow("sample.mzml", RawFileFormat.MzML)] + [DataRow("sample.MZML", RawFileFormat.MzML)] + [DataRow("sample.imzML", RawFileFormat.ImzML)] + [DataRow("sample.IMZML", RawFileFormat.ImzML)] + [DataRow("sample.cdf", RawFileFormat.NetCdf)] + [DataRow("sample.CDF", RawFileFormat.NetCdf)] + [DataRow("sample.lrp", RawFileFormat.KanomaxLrp)] + [DataRow("sample.LRP", RawFileFormat.KanomaxLrp)] + [DataRow("sample.hmd", RawFileFormat.HiveHmd)] + [DataRow("sample.HMD", RawFileFormat.HiveHmd)] + [DataRow("sample.mzb", RawFileFormat.HiveMzb)] + [DataRow("sample.MZB", RawFileFormat.HiveMzb)] + public void AFileIsToldByItsExtension(string name, RawFileFormat expected) { + var missingFolder = Path.Combine(_folder, "not-there"); + + Assert.AreEqual(expected, RawFileFormatClassifier.Classify(Path.Combine(missingFolder, name))); + Assert.IsFalse(Directory.Exists(missingFolder), "classifying must not create anything"); + } + + [DataTestMethod()] + [DataRow("sample.iabf")] + [DataRow("sample.ibd")] + [DataRow("sample.mzXML")] + [DataRow("sample.mzData")] + [DataRow("sample.txt")] + [DataRow("sample.raw.zip")] + [DataRow("sample")] + public void AnyOtherExtensionIsUnknown(string name) { + File.WriteAllBytes(Path.Combine(_folder, name), new byte[0]); + + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(Path.Combine(_folder, name))); + } + + [TestMethod()] + public void NoPathIsUnknown() { + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(null)); + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(string.Empty)); + } + + /// + /// A Waters .raw is a folder and a Thermo .raw a file; a .raw that is not there is neither. + /// + [DataTestMethod()] + [DataRow("sample.raw", "file", RawFileFormat.ThermoRaw)] + [DataRow("sample.RAW", "file", RawFileFormat.ThermoRaw)] + [DataRow("sample.raw", "folder", RawFileFormat.WatersRaw)] + [DataRow("sample.RAW", "folder", RawFileFormat.WatersRaw)] + [DataRow("sample.raw", "missing", RawFileFormat.Unknown)] + public void ADotRawIsToldByWhatItIs(string name, string kind, RawFileFormat expected) { + var path = Path.Combine(_folder, name); + if (kind == "file") { + File.WriteAllBytes(path, new byte[0]); + } + else if (kind == "folder") { + Directory.CreateDirectory(path); + } + + Assert.AreEqual(expected, RawFileFormatClassifier.Classify(path)); + } + + /// + /// Agilent and Bruker both write a .d folder. It is checked in the order RawDataHandler reads it: AcqData first, + /// then analysis.tdf, analysis.tsf and analysis.baf. + /// + [DataTestMethod()] + // content: entries separated by "|", a folder name ending in "/"; "" for an empty .d folder, null for no folder at all. + [DataRow("AcqData/", RawFileFormat.AgilentMassHunterD)] + [DataRow("analysis.tdf", RawFileFormat.BrukerTdf)] + [DataRow("analysis.tsf", RawFileFormat.BrukerTsf)] + [DataRow("analysis.baf", RawFileFormat.BrukerBaf)] + [DataRow("AcqData/|analysis.tdf", RawFileFormat.AgilentMassHunterD)] + [DataRow("AcqData/|analysis.baf", RawFileFormat.AgilentMassHunterD)] + [DataRow("analysis.tdf|analysis.tsf|analysis.baf", RawFileFormat.BrukerTdf)] + [DataRow("analysis.tsf|analysis.baf", RawFileFormat.BrukerTsf)] + [DataRow("AcqData", RawFileFormat.Unknown)] + [DataRow("analysis.tdf/", RawFileFormat.Unknown)] + [DataRow("", RawFileFormat.Unknown)] + [DataRow(null, RawFileFormat.Unknown)] + public void ADotDIsToldByWhatItHolds(string content, RawFileFormat expected) { + foreach (var name in new[] { "sample.d", "SAMPLE2.D" }) { + var folder = Path.Combine(_folder, name); + if (content is not null) { + Directory.CreateDirectory(folder); + foreach (var entry in content.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries)) { + if (entry.EndsWith("/")) { + Directory.CreateDirectory(Path.Combine(folder, entry.TrimEnd('/'))); + } + else { + File.WriteAllBytes(Path.Combine(folder, entry), new byte[0]); + } + } + } + + Assert.AreEqual(expected, RawFileFormatClassifier.Classify(folder), name); + } + } + + [DataTestMethod()] + [DataRow("a\0b.raw")] + [DataRow("a\0b.d")] + [DataRow("ac|d.d")] + [DataRow("a\"b.raw")] + public void APathThatCannotBeCheckedIsUnknown(string name) { + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(name)); + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(Path.Combine(_folder, "x") + name)); + } + + [TestMethod()] + public void AVeryLongPathIsUnknownRatherThanAnError() { + var path = Path.Combine(_folder, new string('x', 40000) + ".raw"); + + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(path)); + } + + [TestMethod()] + public void RecordIfUnknownRecordsWhatTheRawDataTell() { + var path = Path.Combine(_folder, "sample.raw"); + File.WriteAllBytes(path, new byte[0]); + var file = new AnalysisFileBean { AnalysisFilePath = path, }; + + Assert.IsTrue(RawFileFormatClassifier.RecordIfUnknown(file)); + Assert.AreEqual(RawFileFormat.ThermoRaw, file.RawFileFormat); + } + + [TestMethod()] + public void RecordIfUnknownNeverReplacesARecordedFormat() { + var path = Path.Combine(_folder, "sample.raw"); + File.WriteAllBytes(path, new byte[0]); + var file = new AnalysisFileBean { AnalysisFilePath = path, RawFileFormat = RawFileFormat.WatersRaw, }; + + Assert.IsFalse(RawFileFormatClassifier.RecordIfUnknown(file)); + Assert.AreEqual(RawFileFormat.WatersRaw, file.RawFileFormat); + } + + [TestMethod()] + public void RecordIfUnknownLeavesAFileThatCannotBeToldUnknown() { + var file = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.raw"), }; + + Assert.IsFalse(RawFileFormatClassifier.RecordIfUnknown(file)); + Assert.AreEqual(RawFileFormat.Unknown, file.RawFileFormat); + Assert.IsFalse(RawFileFormatClassifier.RecordIfUnknown(null)); + } + + [DataTestMethod()] + [DataRow(RawFileFormat.ThermoRaw, ".raw")] + [DataRow(RawFileFormat.WatersRaw, ".RAW")] + [DataRow(RawFileFormat.AgilentMassHunterD, ".d")] + [DataRow(RawFileFormat.BrukerTdf, ".D")] + [DataRow(RawFileFormat.BrukerTsf, ".d")] + [DataRow(RawFileFormat.BrukerBaf, ".d")] + [DataRow(RawFileFormat.SciexWiff, ".wiff")] + [DataRow(RawFileFormat.SciexWiff2, ".WIFF2")] + [DataRow(RawFileFormat.ShimadzuLcd, ".lcd")] + [DataRow(RawFileFormat.ShimadzuQgd, ".QGD")] + [DataRow(RawFileFormat.MzML, ".mzML")] + [DataRow(RawFileFormat.ImzML, ".imzML")] + [DataRow(RawFileFormat.NetCdf, ".CDF")] + [DataRow(RawFileFormat.Abf, ".abf")] + [DataRow(RawFileFormat.Ibf, ".ibf")] + [DataRow(RawFileFormat.KanomaxLrp, ".lrp")] + [DataRow(RawFileFormat.HiveHmd, ".hmd")] + [DataRow(RawFileFormat.HiveMzb, ".mzb")] + public void AFormatIsConsistentOnlyWithItsOwnExtension(RawFileFormat format, string extension) { + Assert.IsTrue(RawFileFormatClassifier.IsConsistentWith(format, extension)); + Assert.IsTrue(RawFileFormatClassifier.IsConsistentWith(format, extension.ToUpperInvariant())); + Assert.IsTrue(RawFileFormatClassifier.IsConsistentWith(format, extension.ToLowerInvariant())); + foreach (var other in new[] { ".raw", ".d", ".wiff", ".wiff2", ".mzml", ".abf", ".txt", "", null }) { + if (!string.Equals(other, extension, StringComparison.OrdinalIgnoreCase)) { + Assert.IsFalse(RawFileFormatClassifier.IsConsistentWith(format, other), $"{format} with {other ?? "null"}"); + } + } + } + + [TestMethod()] + public void UnknownIsConsistentWithNothing() { + foreach (var extension in new[] { ".raw", ".d", ".mzml", ".txt", "", null }) { + Assert.IsFalse(RawFileFormatClassifier.IsConsistentWith(RawFileFormat.Unknown, extension), extension ?? "null"); + } + } + + /// + /// A format added to the enum without an extension would never be trusted by the exporter. + /// + [TestMethod()] + public void EveryKnownFormatHasAnExtension() { + var extensions = new[] { ".raw", ".d", ".wiff", ".wiff2", ".lcd", ".qgd", ".mzml", ".imzml", ".cdf", ".abf", ".ibf", ".lrp", ".hmd", ".mzb" }; + foreach (var format in System.Enum.GetValues(typeof(RawFileFormat)).Cast().Where(f => f != RawFileFormat.Unknown)) { + Assert.IsTrue(extensions.Any(extension => RawFileFormatClassifier.IsConsistentWith(format, extension)), format.ToString()); + } + } + + [TestMethod()] + public void ARecordedFormatDescribesAFileWhoseRawDataAreGone() { + var file = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.d"), RawFileFormat = RawFileFormat.BrukerTdf, }; + + Assert.AreEqual(RawFileFormat.BrukerTdf, RawFileFormatClassifier.Resolve(file)); + } + + [TestMethod()] + public void ARecordedFormatThatDoesNotFitThePathGivesWayToTheDisk() { + var folder = Path.Combine(_folder, "sample.d"); + Directory.CreateDirectory(folder); + File.WriteAllBytes(Path.Combine(folder, "analysis.tdf"), new byte[0]); + var present = new AnalysisFileBean { AnalysisFilePath = folder, RawFileFormat = RawFileFormat.ThermoRaw, }; + var gone = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.d"), RawFileFormat = RawFileFormat.ThermoRaw, }; + + Assert.AreEqual(RawFileFormat.BrukerTdf, RawFileFormatClassifier.Resolve(present)); + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Resolve(gone)); + } + + [TestMethod()] + public void NothingRecordedIsToldFromTheDisk() { + var path = Path.Combine(_folder, "sample.raw"); + Directory.CreateDirectory(path); + + Assert.AreEqual(RawFileFormat.WatersRaw, RawFileFormatClassifier.Resolve(new AnalysisFileBean { AnalysisFilePath = path, })); + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Resolve(new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.raw"), })); + } +} From 026cfb84bb7f7d721e87e6e93475a6f38bbcfada Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Fri, 16 Oct 2026 00:48:23 +0900 Subject: [PATCH 2/4] Record each raw data file's format when it is registered, and learn it for older projects when the raw data are found The format is recorded wherever MS-DIAL registers a file while it is surely on disk: - GUI: DatasetFileSettingModel.SetFiles, when a project is created. - Console: AnalysisFilesParser, in both bean factories, which covers the folder, CSV and single-file inputs. - GUI: AnalysisFileBeanModel.AnalysisFilePath, when a sample is pointed at a different file. The previous file's format no longer describes it, so the path is classified again. MsdialDataStorage.FixDatasetFolder, a folder move, still leaves the recorded format alone. A project saved before formats were recorded learns them lazily. Every disk check in AnalysisFileBeanModel (IsRawDataAvailable, RefreshRawDataAvailability, which DatasetModel.OpenMethodAsync runs over every file when a dataset opens) goes through one helper. When the raw data are present it records the format if none is recorded yet; the value is kept when the project is next saved. MarkRawDataMissing does not classify, and a recorded format is never replaced by a refresh. DataAccess.LoadMeasurement is deliberately not hooked: the 5.6 line deletes it. No Model class gains a UI or ViewModel reference. Co-Authored-By: Claude Opus 5.5 --- .../Model/DataObj/AnalysisFileBeanModel.cs | 16 ++- .../Model/Setting/DatasetFileSettingModel.cs | 3 + .../Parser/AnalysisFilesParser.cs | 8 +- .../AnalysisFilesParserRawFileFormatTests.cs | 111 ++++++++++++++++ ...AnalysisFileBeanModelRawFileFormatTests.cs | 123 ++++++++++++++++++ .../Setting/DatasetFileSettingModelTests.cs | 51 ++++++++ 6 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 tests/MSDIAL5/MsdialCoreTestAppTests/Parser/AnalysisFilesParserRawFileFormatTests.cs create mode 100644 tests/MSDIAL5/MsdialGuiAppTests/Model/DataObj/AnalysisFileBeanModelRawFileFormatTests.cs create mode 100644 tests/MSDIAL5/MsdialGuiAppTests/Model/Setting/DatasetFileSettingModelTests.cs diff --git a/src/MSDIAL5/MsdialGuiApp/Model/DataObj/AnalysisFileBeanModel.cs b/src/MSDIAL5/MsdialGuiApp/Model/DataObj/AnalysisFileBeanModel.cs index f1287d44e..547c0461a 100644 --- a/src/MSDIAL5/MsdialGuiApp/Model/DataObj/AnalysisFileBeanModel.cs +++ b/src/MSDIAL5/MsdialGuiApp/Model/DataObj/AnalysisFileBeanModel.cs @@ -93,6 +93,8 @@ public string AnalysisFilePath { set { if (_file.AnalysisFilePath != value) { _file.AnalysisFilePath = value; + // Another raw data file: what was recorded for the previous one no longer describes it. + _file.RawFileFormat = RawFileFormatClassifier.Classify(value); OnPropertyChanged(nameof(AnalysisFilePath)); _isRawDataAvailable = null; OnPropertyChanged(nameof(IsRawDataAvailable)); @@ -104,14 +106,24 @@ public string AnalysisFilePath { /// Whether the raw data file of this sample is present, as last checked. A runtime fact, never saved: /// the first read checks the disk, and checks again. /// - public bool IsRawDataAvailable => _isRawDataAvailable ??= DataAccess.RawDataExists(_file.AnalysisFilePath); + public bool IsRawDataAvailable => _isRawDataAvailable ??= CheckRawDataExists(); private bool? _isRawDataAvailable; public bool RefreshRawDataAvailability() { - SetRawDataAvailability(DataAccess.RawDataExists(_file.AnalysisFilePath)); + SetRawDataAvailability(CheckRawDataExists()); return _isRawDataAvailable!.Value; } + // Every check of the disk comes here. A project saved before raw data formats were recorded learns the format + // of a file the first time its raw data are found, and keeps it when the project is saved. + private bool CheckRawDataExists() { + var exists = DataAccess.RawDataExists(_file.AnalysisFilePath); + if (exists) { + RawFileFormatClassifier.RecordIfUnknown(_file); + } + return exists; + } + // The folder the raw data file was in is gone, so the file is too; the collection checks a folder once for all its files. internal void MarkRawDataMissing() { SetRawDataAvailability(false); diff --git a/src/MSDIAL5/MsdialGuiApp/Model/Setting/DatasetFileSettingModel.cs b/src/MSDIAL5/MsdialGuiApp/Model/Setting/DatasetFileSettingModel.cs index d60b44ec9..083e1f6af 100644 --- a/src/MSDIAL5/MsdialGuiApp/Model/Setting/DatasetFileSettingModel.cs +++ b/src/MSDIAL5/MsdialGuiApp/Model/Setting/DatasetFileSettingModel.cs @@ -6,6 +6,7 @@ using CompMs.MsdialCore.DataObj; using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.Parameter; +using CompMs.MsdialCore.Utility; using System; using System.Collections.Generic; using System.IO; @@ -81,6 +82,8 @@ public void SetFiles(IEnumerable files) { ProteinAssembledResultFilePath = Path.Combine(folder, $"{name}_{_dateTime:yyyyMMddHHmm}.{MsdialDataStorageFormat.prf}"), RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean(Path.Combine(folder, $"{name}_{_dateTime:yyyyMMddHHmm}.{MsdialDataStorageFormat.rtc}")), AcquisitionType = AcquisitionType.DDA, + // Recorded while the file is present, so mzTab-M can still name its format after the raw data are gone. + RawFileFormat = RawFileFormatClassifier.Classify(file), }; FileModels.AddAnalysisFile(bean); checker.CheckFileType(bean.AnalysisFilePath); diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/AnalysisFilesParser.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/AnalysisFilesParser.cs index ba21b34e7..d1b0c4b09 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/AnalysisFilesParser.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/AnalysisFilesParser.cs @@ -237,7 +237,9 @@ public static List ReadCsvContents(string filepath) PeakAreaBeanInformationFilePath = Path.Combine(fileDir, afFilename + "_" + dtString + ".pai"), RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean(fileDir + "\\" + afFilename + "_" + dtString + ".rtc"), AnalysisBatch = afBatch, - DilutionFactor = afInjectVolume + DilutionFactor = afInjectVolume, + // Recorded while the file is present, so mzTab-M can still name its format after the raw data are gone. + RawFileFormat = RawFileFormatClassifier.Classify(afFilepath), }); counter++; } @@ -286,7 +288,9 @@ public static List CreateAnalysisFileBeans(List filePa AnalysisFileType = AnalysisFileType.Sample, DeconvolutionFilePath = Path.Combine(fileDir, filename + "_" + dtString + ".dcl"), PeakAreaBeanInformationFilePath = Path.Combine(fileDir, filename + "_" + dtString + ".pai"), - RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean(Path.Combine(fileDir, filename + "_" + dtString + ".rtc")) + RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean(Path.Combine(fileDir, filename + "_" + dtString + ".rtc")), + // Recorded while the file is present, so mzTab-M can still name its format after the raw data are gone. + RawFileFormat = RawFileFormatClassifier.Classify(filepath), }); counter++; } diff --git a/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/AnalysisFilesParserRawFileFormatTests.cs b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/AnalysisFilesParserRawFileFormatTests.cs new file mode 100644 index 000000000..79c71f214 --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/AnalysisFilesParserRawFileFormatTests.cs @@ -0,0 +1,111 @@ +using CompMs.App.MsdialConsole.Parser; +using CompMs.MsdialCore.Enum; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; + +namespace MsdialCoreTestAppTests.Parser; + +/// +/// The Console records each raw data file's format when it reads the analysis files, while the files are surely there, +/// so the format survives in the project after the raw data are deleted. +/// +[TestClass] +public sealed class AnalysisFilesParserRawFileFormatTests +{ + [TestMethod] + public void ReadFolderContents_RecordsEachFormat() + { + using var directory = VendorFolder(); + + var files = AnalysisFilesParser.ReadFolderContents(directory.Path); + + AssertFormats(files.Select(file => (Path.GetFileName(file.AnalysisFilePath), file.RawFileFormat))); + } + + [TestMethod] + public void ReadCsvContents_RecordsEachFormat() + { + using var directory = VendorFolder(); + var names = new[] { "thermo.raw", "waters.raw", "agilent.d", "bruker.d", "sample.mzML", "sample.lcd" }; + var csv = directory.CreateFile( + "analysis_files.csv", + "file_path,file_name,file_type,class_id,acquisition_type,batch_order,analytical_order,factor" + Environment.NewLine + + string.Join(Environment.NewLine, names.Select((name, i) => $"{System.IO.Path.Combine(directory.Path, name)},{Path.GetFileNameWithoutExtension(name)}_{i},Sample,A,DDA,1,{i + 1},1"))); + + var files = AnalysisFilesParser.ReadCsvContents(csv); + + Assert.AreEqual(names.Length, files.Count); + AssertFormats(files.Select(file => (Path.GetFileName(file.AnalysisFilePath), file.RawFileFormat))); + } + + [TestMethod] + public void ReadInput_ASingleFile_RecordsItsFormat() + { + using var directory = new TemporaryDirectory(); + var path = directory.CreateFile("thermo.raw"); + + var files = AnalysisFilesParser.ReadInput(path, new StringReader(string.Empty), new StringWriter(), inputRedirected: true, outputRedirected: true); + + Assert.IsNotNull(files); + Assert.AreEqual(RawFileFormat.ThermoRaw, files.Single().RawFileFormat); + } + + private static void AssertFormats(System.Collections.Generic.IEnumerable<(string Name, RawFileFormat Format)> actual) + { + var formats = actual.ToDictionary(pair => pair.Name, pair => pair.Format); + Assert.AreEqual(RawFileFormat.ThermoRaw, formats["thermo.raw"]); + Assert.AreEqual(RawFileFormat.WatersRaw, formats["waters.raw"]); + Assert.AreEqual(RawFileFormat.AgilentMassHunterD, formats["agilent.d"]); + Assert.AreEqual(RawFileFormat.BrukerTdf, formats["bruker.d"]); + Assert.AreEqual(RawFileFormat.MzML, formats["sample.mzML"]); + Assert.AreEqual(RawFileFormat.ShimadzuLcd, formats["sample.lcd"]); + } + + private static TemporaryDirectory VendorFolder() + { + var directory = new TemporaryDirectory(); + directory.CreateFile("thermo.raw"); + directory.CreateDirectory("waters.raw"); + directory.CreateDirectory(System.IO.Path.Combine("agilent.d", "AcqData")); + directory.CreateDirectory("bruker.d"); + directory.CreateFile(System.IO.Path.Combine("bruker.d", "analysis.tdf")); + directory.CreateFile("sample.mzML"); + directory.CreateFile("sample.lcd"); + return directory; + } + + private sealed class TemporaryDirectory : IDisposable + { + public TemporaryDirectory() + { + Path = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + "MsdialCoreTestAppTests", + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public string CreateDirectory(string name) + { + var path = System.IO.Path.Combine(Path, name); + Directory.CreateDirectory(path); + return path; + } + + public string CreateFile(string name, string content = "") + { + var path = System.IO.Path.Combine(Path, name); + File.WriteAllText(path, content); + return path; + } + + public void Dispose() + { + Directory.Delete(Path, recursive: true); + } + } +} diff --git a/tests/MSDIAL5/MsdialGuiAppTests/Model/DataObj/AnalysisFileBeanModelRawFileFormatTests.cs b/tests/MSDIAL5/MsdialGuiAppTests/Model/DataObj/AnalysisFileBeanModelRawFileFormatTests.cs new file mode 100644 index 000000000..8b8f5f734 --- /dev/null +++ b/tests/MSDIAL5/MsdialGuiAppTests/Model/DataObj/AnalysisFileBeanModelRawFileFormatTests.cs @@ -0,0 +1,123 @@ +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; + +namespace CompMs.App.Msdial.Model.DataObj.Tests +{ + /// + /// A project saved before raw data formats were recorded learns each file's format the first time the GUI finds its + /// raw data, and a file pointed at other raw data is described by those. + /// + [TestClass()] + public class AnalysisFileBeanModelRawFileFormatTests + { + private string _folder = string.Empty; + + [TestInitialize()] + public void Setup() { + _folder = Path.Combine(Path.GetTempPath(), $"msdial-analysisfile-format-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_folder); + } + + [TestCleanup()] + public void Cleanup() { + Directory.Delete(_folder, recursive: true); + } + + [TestMethod()] + public void RefreshingFindsAndRecordsTheFormatOfAnOldProjectsFile() { + var bean = new AnalysisFileBean { AnalysisFilePath = CreateThermoRaw("sample.raw"), }; + var model = new AnalysisFileBeanModel(bean); + + Assert.IsTrue(model.RefreshRawDataAvailability()); + + Assert.AreEqual(RawFileFormat.ThermoRaw, bean.RawFileFormat); + } + + [TestMethod()] + public void TheFirstCheckOfTheRawDataAlsoRecordsTheFormat() { + var folder = Path.Combine(_folder, "sample.d"); + Directory.CreateDirectory(folder); + File.WriteAllBytes(Path.Combine(folder, "analysis.tdf"), new byte[0]); + var bean = new AnalysisFileBean { AnalysisFilePath = folder, }; + var model = new AnalysisFileBeanModel(bean); + + Assert.IsTrue(model.IsRawDataAvailable); + + Assert.AreEqual(RawFileFormat.BrukerTdf, bean.RawFileFormat); + } + + [TestMethod()] + public void AFileWhoseRawDataAreMissingStaysUnknown() { + var bean = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.raw"), }; + var model = new AnalysisFileBeanModel(bean); + + Assert.IsFalse(model.RefreshRawDataAvailability()); + Assert.IsFalse(model.IsRawDataAvailable); + + Assert.AreEqual(RawFileFormat.Unknown, bean.RawFileFormat); + } + + [TestMethod()] + public void ARecordedFormatIsNotReplacedByARefresh() { + var bean = new AnalysisFileBean { AnalysisFilePath = CreateThermoRaw("sample.raw"), RawFileFormat = RawFileFormat.WatersRaw, }; + var model = new AnalysisFileBeanModel(bean); + + Assert.IsTrue(model.RefreshRawDataAvailability()); + + Assert.AreEqual(RawFileFormat.WatersRaw, bean.RawFileFormat); + } + + [TestMethod()] + public void PointingAtOtherRawDataDescribesThem() { + var bean = new AnalysisFileBean { AnalysisFilePath = CreateThermoRaw("thermo.raw"), RawFileFormat = RawFileFormat.ThermoRaw, }; + var model = new AnalysisFileBeanModel(bean); + var waters = Path.Combine(_folder, "waters.raw"); + Directory.CreateDirectory(waters); + + model.AnalysisFilePath = waters; + Assert.AreEqual(RawFileFormat.WatersRaw, bean.RawFileFormat); + + model.AnalysisFilePath = Path.Combine(_folder, "sample.mzML"); + Assert.AreEqual(RawFileFormat.MzML, bean.RawFileFormat); + + // What was recorded for the previous file does not describe one that cannot be told. + model.AnalysisFilePath = Path.Combine(_folder, "gone.d"); + Assert.AreEqual(RawFileFormat.Unknown, bean.RawFileFormat); + } + + [TestMethod()] + public void SettingTheSamePathKeepsTheRecordedFormat() { + var path = Path.Combine(_folder, "gone.raw"); + var bean = new AnalysisFileBean { AnalysisFilePath = path, RawFileFormat = RawFileFormat.ThermoRaw, }; + var model = new AnalysisFileBeanModel(bean); + + model.AnalysisFilePath = path; + + Assert.AreEqual(RawFileFormat.ThermoRaw, bean.RawFileFormat); + } + + [TestMethod()] + public void MarkingTheRawDataMissingKeepsTheRecordedFormat() { + var bean = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone", "sample.d"), RawFileFormat = RawFileFormat.BrukerTdf, }; + var model = new AnalysisFileBeanModel(bean); + + model.MarkRawDataMissing(); + Assert.AreEqual(RawFileFormat.BrukerTdf, bean.RawFileFormat); + + // The collection marks the samples of a folder that is gone without checking each of them. + var collection = new AnalysisFileBeanModelCollection(new[] { model }); + CollectionAssert.AreEqual(new[] { model }, collection.RefreshRawDataAvailability().ToArray()); + Assert.AreEqual(RawFileFormat.BrukerTdf, bean.RawFileFormat); + } + + private string CreateThermoRaw(string name) { + var path = Path.Combine(_folder, name); + File.WriteAllBytes(path, new byte[0]); + return path; + } + } +} diff --git a/tests/MSDIAL5/MsdialGuiAppTests/Model/Setting/DatasetFileSettingModelTests.cs b/tests/MSDIAL5/MsdialGuiAppTests/Model/Setting/DatasetFileSettingModelTests.cs new file mode 100644 index 000000000..ac2520868 --- /dev/null +++ b/tests/MSDIAL5/MsdialGuiAppTests/Model/Setting/DatasetFileSettingModelTests.cs @@ -0,0 +1,51 @@ +using CompMs.MsdialCore.Enum; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; + +namespace CompMs.App.Msdial.Model.Setting.Tests +{ + [TestClass()] + public class DatasetFileSettingModelTests + { + private string _folder = string.Empty; + + [TestInitialize()] + public void Setup() { + _folder = Path.Combine(Path.GetTempPath(), $"msdial-datasetfilesetting-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_folder); + } + + [TestCleanup()] + public void Cleanup() { + Directory.Delete(_folder, recursive: true); + } + + /// + /// The files are registered while they are present, so their formats are recorded then, for mzTab-M to name + /// after the raw data are gone. + /// + [TestMethod()] + public void SetFilesRecordsTheFormatOfEachFile() { + var thermo = Path.Combine(_folder, "a_thermo.raw"); + File.WriteAllBytes(thermo, new byte[0]); + var waters = Path.Combine(_folder, "b_waters.raw"); + Directory.CreateDirectory(waters); + var agilent = Path.Combine(_folder, "c_agilent.d"); + Directory.CreateDirectory(Path.Combine(agilent, "AcqData")); + var bruker = Path.Combine(_folder, "d_bruker.d"); + Directory.CreateDirectory(bruker); + File.WriteAllBytes(Path.Combine(bruker, "analysis.baf"), new byte[0]); + var mzml = Path.Combine(_folder, "e_sample.mzML"); + File.WriteAllBytes(mzml, new byte[0]); + var model = new DatasetFileSettingModel(new DateTime(2026, 10, 16)); + + model.SetFiles(new[] { mzml, bruker, agilent, waters, thermo, }); + + CollectionAssert.AreEqual( + new[] { RawFileFormat.ThermoRaw, RawFileFormat.WatersRaw, RawFileFormat.AgilentMassHunterD, RawFileFormat.BrukerBaf, RawFileFormat.MzML, }, + model.FileModels.AnalysisFiles.Select(file => file.File.RawFileFormat).ToArray()); + } + } +} From aabbb940830128a48adda25eb784483eb14479b3 Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Fri, 16 Oct 2026 00:48:35 +0900 Subject: [PATCH 3/4] Write the recorded raw data format as mzTab-M ms_run-format, so it survives the raw data SetRawFileMetadataDic now describes each run by, in order: 1. the format recorded on the AnalysisFileBean, when it fits the file's extension; 2. otherwise what RawFileFormatClassifier tells from the disk now, which is the check the switch used to make inline and covers projects saved before formats were recorded; 3. otherwise the same Unknown user terms as before: "Unknown .raw format", "Unknown .d folder format", "Unknown file format". The terms move unchanged into MsRunFormatTerms(format, extension), including the nativeID terms that the "MS-DIAL set Datapoint Number" id_format overrides. AnalysisFileExtention, and with it the EDAM CV declaration for .CDF, is untouched. imzML still has no term here. The term corrections for WIFF2, imzML and netCDF are left for a separate change. ms_run-format is byte-identical for every file present on disk, with one exception: a .d holding both AcqData and a Bruker analysis file is now Agilent's, the format RawDataHandler reads it as. It used to be described by the Bruker file. A differential run of the old switch against the new path (74 present and missing paths, every extension, letter case and .raw/.d shape) found no other difference. Tests: - a recorded format is written when the raw data are gone, checked against the PSI-MS file-format children (psi-ms.obo 4.2.2); - a recorded format that does not fit the path is ignored; - every PSI-MS term the exporter can write is a file format; - the .d holding both kinds of file; - end to end: record while the raw data are present, save the project, delete the raw data, reload, export, and assert the terms. Exporting creates no path. The existing .d and .raw tests keep their expectations; their summaries now say they describe the case where nothing is recorded. Co-Authored-By: Claude Opus 5.5 --- .../MsdialCore/Export/MztabFormatExport.cs | 171 +++++------- .../Export/MztabFormatExportTests.cs | 255 +++++++++++++++++- 2 files changed, 324 insertions(+), 102 deletions(-) diff --git a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs index 4b2f5994b..7bc05587a 100644 --- a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs +++ b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs @@ -4,8 +4,10 @@ using CompMs.Common.Utility; using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.MSDec; using CompMs.MsdialCore.Parameter; +using CompMs.MsdialCore.Utility; using System; using System.Collections.Generic; using System.IO; @@ -1555,104 +1557,10 @@ private static IReadOnlyDictionary SetRawFileMetadataDic(I var analysisFilePath = files[i].AnalysisFilePath; var analysisFileExtention = Path.GetExtension(analysisFilePath).ToUpper(); - var msRunFormat = ""; // analysed file format - var msRunIDFormat = ""; // analysed file Datapoint Number - switch (analysisFileExtention) - { - case (".ABF"): - msRunFormat = "[,, ABF(Analysis Base File) file, ]"; - msRunIDFormat = "[,, ABF file Datapoint Number, ]"; - break; - case (".IBF"): - msRunFormat = "[,, IBF file, ]"; - msRunIDFormat = "[,, IBF file Datapoint Number, ]"; - break; - case (".WIFF"): - case (".WIFF2"): - msRunFormat = "[MS, MS:1000562, ABI WIFF format, ]"; - msRunIDFormat = "[MS, MS:1000770, WIFF nativeID format, ]"; - break; - case (".D"): - // Agilent and Bruker both write a ".d" folder. Bruker's holds analysis.baf - // (BAF) or analysis.tdf / analysis.tsf (timsTOF); Agilent's holds AcqData. - if (File.Exists(Path.Combine(analysisFilePath, "analysis.tdf"))) - { - msRunFormat = "[MS, MS:1002817, Bruker TDF format, ]"; - msRunIDFormat = "[MS, MS:1002818, Bruker TDF nativeID format, ]"; - } - else if (File.Exists(Path.Combine(analysisFilePath, "analysis.tsf"))) - { - msRunFormat = "[MS, MS:1003282, Bruker TSF format, ]"; - msRunIDFormat = "[MS, MS:1003283, Bruker TSF nativeID format, ]"; - } - else if (File.Exists(Path.Combine(analysisFilePath, "analysis.baf"))) - { - msRunFormat = "[MS, MS:1000815, Bruker BAF format, ]"; - msRunIDFormat = "[MS, MS:1000772, Bruker BAF nativeID format, ]"; - } - else if (Directory.Exists(Path.Combine(analysisFilePath, "AcqData"))) - { - msRunFormat = "[MS, MS:1001509, Agilent MassHunter format, ]"; - msRunIDFormat = "[MS, MS:1001508, Agilent MassHunter nativeID format, ]"; - } - else - { - msRunFormat = "[,, Unknown .d folder format, ]"; - msRunIDFormat = "[,, Unknown file format Datapoint Number, ]"; - } - break; - case (".CDF"): - msRunFormat = "[EDAM, format:3650, netCDF, ]"; - msRunIDFormat = "[MS, MS:1000776, scan number only nativeID format, ]"; - break; - case (".MZML"): - msRunFormat = "[MS, MS:1000584, mzML format, ]"; - msRunIDFormat = "[MS, MS:1000776, scan number only nativeID format, ]"; - break; - case (".RAW"): - // A Waters .raw is a folder and a Thermo .raw a file; a .raw that is not there is neither, - // and is described as unknown, as a .d folder that holds nothing recognisable is. - if (Directory.Exists(analysisFilePath)) - { - msRunFormat = "[MS, MS:1000526, Waters raw format, ]"; - msRunIDFormat = "[MS, MS:1000769, Waters nativeID format, ]"; - } - else if (File.Exists(analysisFilePath)) - { - msRunFormat = "[MS, MS:1000563, Thermo RAW format, ]"; - msRunIDFormat = "[MS, MS:1000768, Thermo nativeID format, ]"; - } - else - { - msRunFormat = "[,, Unknown .raw format, ]"; - msRunIDFormat = "[,, Unknown file format Datapoint Number, ]"; - } - break; - case (".LRP"): - msRunFormat = "[,, LRP file, ]"; - msRunIDFormat = "[,, LRP file Datapoint Number, ]"; - break; - case (".HMD"): - msRunFormat = "[,, Hive HMD file, ]"; - msRunIDFormat = "[,, Hive HMD file Datapoint Number, ]"; - break; - case (".MZB"): - msRunFormat = "[,, Hive mzB file, ]"; - msRunIDFormat = "[,, Hive mzB file Datapoint Number, ]"; - break; - case (".LCD"): - msRunFormat = "[MS, MS:1003009, Shimadzu Biotech LCD format, ]"; - msRunIDFormat = "[MS, MS:1000929, Shimadzu Biotech nativeID format, ]"; - break; - case (".QGD"): - msRunFormat = "[,, Shimadzu GC/MS format, ]"; - msRunIDFormat = "[,, Shimadzu GC/MS format file Datapoint Number, ]"; - break; - default: - msRunFormat = "[,, Unknown file format, ]"; - msRunIDFormat = "[,, Unknown file format Datapoint Number, ]"; - break; - } + // The format recorded while the raw data were present comes first, so a file whose raw data have since + // been deleted or moved is still described by what it was; a file with nothing recorded (a project saved + // before formats were recorded) is described by what the disk shows now. + var (msRunFormat, msRunIDFormat) = MsRunFormatTerms(RawFileFormatClassifier.Resolve(files[i]), analysisFileExtention); msRunIDFormat = "[,, MS-DIAL set Datapoint Number, ]"; var id = files[i].AnalysisFileId + 1; fileMetadataDic.Add(id, new RawFileMetadata() @@ -1676,6 +1584,73 @@ private static IReadOnlyDictionary SetRawFileMetadataDic(I return fileMetadataDic; } + /// + /// The ms_run format and nativeID format terms for a raw data file of . An unknown + /// format is described by a user term that names only the extension (, upper case). + /// + /// + /// These are the terms the exporter wrote when it told the format from the file on disk, kept as they were. + /// The nativeID format is not written: every run's id_format is "MS-DIAL set Datapoint Number", because + /// spectra_ref uses MS-DIAL's scan numbers. imzML has no term here and is written as an unknown file. + /// + internal static (string Format, string IdFormat) MsRunFormatTerms(RawFileFormat format, string upperExtension) + { + switch (format) + { + case RawFileFormat.Abf: + return ("[,, ABF(Analysis Base File) file, ]", "[,, ABF file Datapoint Number, ]"); + case RawFileFormat.Ibf: + return ("[,, IBF file, ]", "[,, IBF file Datapoint Number, ]"); + case RawFileFormat.SciexWiff: + case RawFileFormat.SciexWiff2: + return ("[MS, MS:1000562, ABI WIFF format, ]", "[MS, MS:1000770, WIFF nativeID format, ]"); + // Agilent and Bruker both write a ".d" folder. Bruker's holds analysis.baf + // (BAF) or analysis.tdf / analysis.tsf (timsTOF); Agilent's holds AcqData. + case RawFileFormat.BrukerTdf: + return ("[MS, MS:1002817, Bruker TDF format, ]", "[MS, MS:1002818, Bruker TDF nativeID format, ]"); + case RawFileFormat.BrukerTsf: + return ("[MS, MS:1003282, Bruker TSF format, ]", "[MS, MS:1003283, Bruker TSF nativeID format, ]"); + case RawFileFormat.BrukerBaf: + return ("[MS, MS:1000815, Bruker BAF format, ]", "[MS, MS:1000772, Bruker BAF nativeID format, ]"); + case RawFileFormat.AgilentMassHunterD: + return ("[MS, MS:1001509, Agilent MassHunter format, ]", "[MS, MS:1001508, Agilent MassHunter nativeID format, ]"); + case RawFileFormat.NetCdf: + return ("[EDAM, format:3650, netCDF, ]", "[MS, MS:1000776, scan number only nativeID format, ]"); + case RawFileFormat.MzML: + return ("[MS, MS:1000584, mzML format, ]", "[MS, MS:1000776, scan number only nativeID format, ]"); + // A Waters .raw is a folder and a Thermo .raw a file. + case RawFileFormat.WatersRaw: + return ("[MS, MS:1000526, Waters raw format, ]", "[MS, MS:1000769, Waters nativeID format, ]"); + case RawFileFormat.ThermoRaw: + return ("[MS, MS:1000563, Thermo RAW format, ]", "[MS, MS:1000768, Thermo nativeID format, ]"); + case RawFileFormat.KanomaxLrp: + return ("[,, LRP file, ]", "[,, LRP file Datapoint Number, ]"); + case RawFileFormat.HiveHmd: + return ("[,, Hive HMD file, ]", "[,, Hive HMD file Datapoint Number, ]"); + case RawFileFormat.HiveMzb: + return ("[,, Hive mzB file, ]", "[,, Hive mzB file Datapoint Number, ]"); + case RawFileFormat.ShimadzuLcd: + return ("[MS, MS:1003009, Shimadzu Biotech LCD format, ]", "[MS, MS:1000929, Shimadzu Biotech nativeID format, ]"); + case RawFileFormat.ShimadzuQgd: + return ("[,, Shimadzu GC/MS format, ]", "[,, Shimadzu GC/MS format file Datapoint Number, ]"); + case RawFileFormat.ImzML: + case RawFileFormat.Unknown: + default: + break; + } + // A .raw or .d whose format was never recorded and that is not on disk (or holds nothing recognisable) + // is neither vendor's, and is described as unknown rather than as a vendor it may not be. + switch (upperExtension) + { + case (".RAW"): + return ("[,, Unknown .raw format, ]", "[,, Unknown file format Datapoint Number, ]"); + case (".D"): + return ("[,, Unknown .d folder format, ]", "[,, Unknown file format Datapoint Number, ]"); + default: + return ("[,, Unknown file format, ]", "[,, Unknown file format Datapoint Number, ]"); + } + } + static string SetAdductTypeString(string adductIons) { if (adductIons != "null" && adductIons.Length > 2 && adductIons.Substring(adductIons.Length - 2, 1) == "]") diff --git a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs index 6a848ea43..ee01a0223 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs @@ -4,8 +4,10 @@ using CompMs.Common.Utility; using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.Parameter; using CompMs.MsdialCore.Parser; +using CompMs.MsdialCore.Utility; using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Collections.Generic; using System.IO; @@ -173,8 +175,8 @@ private static string[] ExportToLines( /// /// Agilent and Bruker both write a ".d" folder, and ms_run-format is told apart by what the folder holds. - /// A folder that is not on disk when the file is exported (moved, archived, or a project opened on another - /// machine) gets a neutral user term, not a vendor it may not be. + /// When no format was recorded for it, a folder that is not on disk when the file is exported (moved, archived, + /// or a project opened on another machine) gets a neutral user term, not a vendor it may not be. /// /// /// ms_run-id_format is not checked: SetRawFileMetadataDic overwrites it with "MS-DIAL set Datapoint Number" @@ -229,8 +231,9 @@ public async Task ADotDFolderIsDescribedByWhatItHolds(string content, string exp } /// - /// A .raw is a Waters folder or a Thermo file. One that is not on disk when the file is exported used to throw - /// from File.GetAttributes; it gets a neutral user term, as an unrecognisable .d folder does. + /// A .raw is a Waters folder or a Thermo file. When no format was recorded for it, one that is not on disk when + /// the file is exported (which used to throw from File.GetAttributes) gets a neutral user term, as an + /// unrecognisable .d folder does. /// [DataTestMethod()] [DataRow("folder", "[MS, MS:1000526, Waters raw format, ]")] @@ -272,6 +275,250 @@ public async Task ADotRawPathIsDescribedByWhatItIs(string kind, string expectedF } } + /// + /// A .d folder is read as RawDataHandler reads it, AcqData first, so one that holds both an AcqData folder and a + /// Bruker analysis file is Agilent's. This is the one case whose ms_run-format changed when formats began to be + /// recorded: it used to be described by the Bruker file. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task ADotDFolderHoldingAcqDataIsAgilentsWhateverElseItHolds() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var scratch = Path.Combine(Path.GetTempPath(), $"msdial-mztab-dotd-both-{System.Guid.NewGuid():N}"); + try { + foreach (var file in storage.AnalysisFiles) { + var folder = Path.Combine(scratch, file.AnalysisFileName + ".d"); + Directory.CreateDirectory(Path.Combine(folder, "AcqData")); + File.WriteAllBytes(Path.Combine(folder, "analysis.tdf"), new byte[0]); + file.AnalysisFilePath = folder; + } + + var formats = MsRunFormats(ExportToLines(storage, container, msdecs)); + + Assert.AreEqual(storage.AnalysisFiles.Count, formats.Length); + Assert.IsTrue(formats.All(format => format == "[MS, MS:1001509, Agilent MassHunter format, ]"), string.Join(" | ", formats)); + } + finally { + Directory.Delete(scratch, recursive: true); + } + } + + /// + /// A format recorded while the raw data were present is what ms_run-format says after they are gone, with the + /// term the exporter writes for that format on disk. + /// + [DataTestMethod()] + [DataRow(RawFileFormat.ThermoRaw, ".raw", "[MS, MS:1000563, Thermo RAW format, ]")] + [DataRow(RawFileFormat.WatersRaw, ".raw", "[MS, MS:1000526, Waters raw format, ]")] + [DataRow(RawFileFormat.AgilentMassHunterD, ".d", "[MS, MS:1001509, Agilent MassHunter format, ]")] + [DataRow(RawFileFormat.BrukerTdf, ".d", "[MS, MS:1002817, Bruker TDF format, ]")] + [DataRow(RawFileFormat.BrukerTsf, ".d", "[MS, MS:1003282, Bruker TSF format, ]")] + [DataRow(RawFileFormat.BrukerBaf, ".d", "[MS, MS:1000815, Bruker BAF format, ]")] + [DataRow(RawFileFormat.SciexWiff, ".wiff", "[MS, MS:1000562, ABI WIFF format, ]")] + [DataRow(RawFileFormat.ShimadzuLcd, ".lcd", "[MS, MS:1003009, Shimadzu Biotech LCD format, ]")] + [DataRow(RawFileFormat.MzML, ".mzML", "[MS, MS:1000584, mzML format, ]")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task ARecordedFormatIsWrittenWhenTheRawDataAreGone(RawFileFormat recorded, string extension, string expectedFormat) { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var gone = Path.Combine(Path.GetTempPath(), $"msdial-mztab-gone-{System.Guid.NewGuid():N}"); + foreach (var file in storage.AnalysisFiles) { + file.AnalysisFilePath = Path.Combine(gone, file.AnalysisFileName + extension); + file.RawFileFormat = recorded; + } + + var formats = MsRunFormats(ExportToLines(storage, container, msdecs)); + + Assert.AreEqual(storage.AnalysisFiles.Count, formats.Length); + Assert.IsTrue(formats.All(format => format == expectedFormat), string.Join(" | ", formats)); + foreach (var format in formats) { + AssertIsAPsiMsFileFormat(format); + } + Assert.IsFalse(Directory.Exists(gone), "exporting must not create the raw data folder"); + } + + /// + /// A recorded format that cannot be the file at the path (a Thermo format on a .d) is not trusted; the file is + /// described by what the disk shows, as when nothing is recorded. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task ARecordedFormatThatDoesNotMatchThePathIsIgnored() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var scratch = Path.Combine(Path.GetTempPath(), $"msdial-mztab-mismatch-{System.Guid.NewGuid():N}"); + Directory.CreateDirectory(scratch); + try { + Assert.AreEqual(2, storage.AnalysisFiles.Count); + // Present: a Bruker TDF folder recorded as Thermo. Gone: a .d recorded as Thermo. + var present = Path.Combine(scratch, "present.d"); + Directory.CreateDirectory(present); + File.WriteAllBytes(Path.Combine(present, "analysis.tdf"), new byte[0]); + storage.AnalysisFiles[0].AnalysisFilePath = present; + storage.AnalysisFiles[0].RawFileFormat = RawFileFormat.ThermoRaw; + storage.AnalysisFiles[1].AnalysisFilePath = Path.Combine(scratch, "gone.d"); + storage.AnalysisFiles[1].RawFileFormat = RawFileFormat.ThermoRaw; + + var formats = MsRunFormats(ExportToLines(storage, container, msdecs)); + + CollectionAssert.AreEqual(new[] { "[MS, MS:1002817, Bruker TDF format, ]", "[,, Unknown .d folder format, ]" }, formats); + } + finally { + Directory.Delete(scratch, recursive: true); + } + } + + /// + /// Registration to export: the format is recorded while the raw data are present, the project is saved, the raw + /// data are deleted, and the project reloaded from what was saved still names each format with its PSI-MS term. + /// + [DataTestMethod()] + [DataRow("thermo", "waters")] + [DataRow("bruker-tdf", "agilent")] + [DataRow("bruker-tsf", "bruker-baf")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task AFormatRecordedAtRegistrationOutlivesTheRawData(string firstKind, string secondKind) { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + Assert.AreEqual(2, storage.AnalysisFiles.Count); + var kinds = new[] { firstKind, secondKind }; + var scratch = Path.Combine(Path.GetTempPath(), $"msdial-mztab-e2e-{System.Guid.NewGuid():N}"); + var expected = new string[kinds.Length]; + try { + for (int i = 0; i < kinds.Length; i++) { + var file = storage.AnalysisFiles[i]; + (file.AnalysisFilePath, expected[i]) = CreateRawData(scratch, file.AnalysisFileName, kinds[i]); + Assert.AreEqual(RawFileFormat.Unknown, file.RawFileFormat, "the fixture predates recorded formats"); + Assert.IsTrue(RawFileFormatClassifier.RecordIfUnknown(file), kinds[i]); + } + } + finally { + // The project is saved and exported only after the raw data are gone. + Directory.Delete(scratch, recursive: true); + } + + var memory = new MemoryStream(); + using (IStreamManager manager = ZipStreamManager.OpenCreate(memory)) { + await storage.SaveAsync(manager, "Test", "TestFolder"); + manager.Complete(); + } + IMsdialDataStorage reloaded; + using (var manager = ZipStreamManager.OpenGet(memory)) { + reloaded = await MsdialDataStorage.Serializer.LoadAsync(manager, "Test", null, "TestFolder"); + } + + var formats = MsRunFormats(ExportToLines(reloaded, container, msdecs)); + + CollectionAssert.AreEqual(expected, formats); + foreach (var format in formats) { + AssertIsAPsiMsFileFormat(format); + } + Assert.IsFalse(Directory.Exists(scratch), "exporting must not create the raw data folder"); + } + + /// + /// Every PSI-MS term the exporter can write as ms_run-format, for any recorded format, is a file format term. + /// + [TestMethod()] + public void EveryPsiMsFormatTermIsAFileFormat() { + var written = 0; + foreach (var format in System.Enum.GetValues(typeof(RawFileFormat)).Cast()) { + foreach (var extension in new[] { ".RAW", ".D", ".MZML", ".TXT" }) { + var term = MztabFormatExporter.MsRunFormatTerms(format, extension).Format; + if (term.StartsWith("[MS,")) { + AssertIsAPsiMsFileFormat(term); + written++; + } + } + } + Assert.IsTrue(written > 0); + } + + /// + /// The children of MS:1000560 (mass spectrometer file format) that the exporter writes, with their names as in + /// psi-ms.obo (checked against data-version 4.2.2, where none of them is obsolete). + /// + private static readonly Dictionary PsiMsFileFormats = new Dictionary + { + ["MS:1000526"] = "Waters raw format", + ["MS:1000562"] = "ABI WIFF format", + ["MS:1000563"] = "Thermo RAW format", + ["MS:1000584"] = "mzML format", + ["MS:1000815"] = "Bruker BAF format", + ["MS:1001509"] = "Agilent MassHunter format", + ["MS:1002817"] = "Bruker TDF format", + ["MS:1003009"] = "Shimadzu Biotech LCD format", + ["MS:1003282"] = "Bruker TSF format", + }; + + private static void AssertIsAPsiMsFileFormat(string term) { + // An mzTab-M parameter: [CV label, accession, name, value]. + var fields = term.TrimStart('[').TrimEnd(']').Split(',').Select(field => field.Trim()).ToArray(); + Assert.AreEqual(4, fields.Length, term); + Assert.AreEqual("MS", fields[0], term); + Assert.IsTrue(PsiMsFileFormats.TryGetValue(fields[1], out var name), $"{fields[1]} is not a known file format term: {term}"); + Assert.AreEqual(name, fields[2], term); + Assert.AreEqual(string.Empty, fields[3], term); + } + + private static (string Path, string ExpectedFormat) CreateRawData(string folder, string name, string kind) { + switch (kind) { + case "thermo": { + var path = Path.Combine(folder, name + ".raw"); + Directory.CreateDirectory(folder); + File.WriteAllBytes(path, new byte[0]); + return (path, "[MS, MS:1000563, Thermo RAW format, ]"); + } + case "waters": { + var path = Path.Combine(folder, name + ".raw"); + Directory.CreateDirectory(path); + return (path, "[MS, MS:1000526, Waters raw format, ]"); + } + case "agilent": { + var path = Path.Combine(folder, name + ".d"); + Directory.CreateDirectory(Path.Combine(path, "AcqData")); + return (path, "[MS, MS:1001509, Agilent MassHunter format, ]"); + } + case "bruker-tdf": + case "bruker-tsf": + case "bruker-baf": { + var path = Path.Combine(folder, name + ".d"); + Directory.CreateDirectory(path); + File.WriteAllBytes(Path.Combine(path, "analysis." + kind.Substring("bruker-".Length)), new byte[0]); + var expected = kind == "bruker-tdf" ? "[MS, MS:1002817, Bruker TDF format, ]" + : kind == "bruker-tsf" ? "[MS, MS:1003282, Bruker TSF format, ]" + : "[MS, MS:1000815, Bruker BAF format, ]"; + return (path, expected); + } + default: + throw new System.ArgumentException(kind); + } + } + + private static string[] MsRunFormats(string[] lines) { + return lines.Select(line => line.Split('\t')) + .Where(f => f[0] == "MTD" && f.Length > 2 && f[1].StartsWith("ms_run[") && f[1].EndsWith("]-format")) + .Select(f => f[2]) + .ToArray(); + } + [TestMethod()] //[DeploymentItem(@"Resources\Export\small_test_project.mdproject", @"Resources\Export")] From 6394a3e57f078725f7b7c640e883a7640442a514 Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Fri, 16 Oct 2026 01:09:17 +0900 Subject: [PATCH 4/4] Describe raw data on disk by what is there, and use the recorded format only when the disk cannot tell Resolve preferred a recorded format over the disk whenever it fitted the extension. A format is recorded once and never replaced, so if another file of the same extension later takes the recorded one's place (a Waters .raw folder where a Thermo .raw file was, or a Bruker .d where an Agilent one was), ms_run-format named the old format for raw data that are still on disk. Master names what is there, so this changed output for a file that is present, which the change was not to do. Resolve now asks the disk first and falls back to the recorded format only when the disk cannot tell, that is when the raw data are deleted or moved (or a vendor folder no longer holds its marker file). A recorded format that does not fit the extension is still ignored. Output for every file on disk is again what the classifier tells. The one difference from before formats were recorded is still the .d holding both AcqData and a Bruker analysis file, which is now Agilent's. The recorded format is what makes the difference once the raw data are gone, as before. Tests: - Resolve describes a .raw or .d on disk by its contents whatever was recorded, in both directions, in any letter case, and leaves the record untouched; - an emptied .d folder is described by its record; - an export describes a Waters folder recorded as Thermo, and a Thermo file recorded as Waters, by what is on disk. The new cases fail against the previous Resolve. Co-Authored-By: Claude Opus 5.5 --- .../MsdialCore/Export/MztabFormatExport.cs | 6 +-- .../Utility/RawFileFormatClassifier.cs | 19 ++++++--- .../Export/MztabFormatExportTests.cs | 38 +++++++++++++++++ .../Utility/RawFileFormatClassifierTests.cs | 42 +++++++++++++++++++ 4 files changed, 96 insertions(+), 9 deletions(-) diff --git a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs index 7bc05587a..dae52cb12 100644 --- a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs +++ b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs @@ -1557,9 +1557,9 @@ private static IReadOnlyDictionary SetRawFileMetadataDic(I var analysisFilePath = files[i].AnalysisFilePath; var analysisFileExtention = Path.GetExtension(analysisFilePath).ToUpper(); - // The format recorded while the raw data were present comes first, so a file whose raw data have since - // been deleted or moved is still described by what it was; a file with nothing recorded (a project saved - // before formats were recorded) is described by what the disk shows now. + // A file on disk is described by what the disk shows, as before formats were recorded. A file whose raw + // data have since been deleted or moved is described by the format recorded while they were present; with + // nothing recorded (a project saved before formats were recorded) it gets the unknown terms. var (msRunFormat, msRunIDFormat) = MsRunFormatTerms(RawFileFormatClassifier.Resolve(files[i]), analysisFileExtention); msRunIDFormat = "[,, MS-DIAL set Datapoint Number, ]"; var id = files[i].AnalysisFileId + 1; diff --git a/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs b/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs index 8bceaae74..5b485a4d9 100644 --- a/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs +++ b/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs @@ -76,15 +76,22 @@ public static bool IsConsistentWith(RawFileFormat format, string extension) { } /// - /// The format to describe by: the recorded one when it fits the file's extension, which - /// holds after the raw data are gone; otherwise what tells from the disk now. + /// The format to describe by: what tells from the disk now, and when + /// the disk cannot tell (the raw data are deleted or moved), the recorded format when it fits the file's extension. /// + /// + /// The disk comes first because a format is recorded once and never replaced: if another file of the same extension + /// later takes the place of the recorded one (a Waters .raw folder where a Thermo .raw file was), the record no + /// longer describes what is at the path. A file on disk is therefore described by what is there, and the record is + /// used only for what the disk can no longer tell. + /// public static RawFileFormat Resolve(AnalysisFileBean file) { - var recorded = file.RawFileFormat; - if (recorded != RawFileFormat.Unknown && IsConsistentWith(recorded, ExtensionOrNull(file.AnalysisFilePath))) { - return recorded; + var onDisk = Classify(file.AnalysisFilePath); + if (onDisk != RawFileFormat.Unknown) { + return onDisk; } - return Classify(file.AnalysisFilePath); + var recorded = file.RawFileFormat; + return IsConsistentWith(recorded, ExtensionOrNull(file.AnalysisFilePath)) ? recorded : RawFileFormat.Unknown; } private static RawFileFormat ClassifyDotRaw(string path) { diff --git a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs index ee01a0223..d4c173e5b 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs @@ -381,6 +381,44 @@ public async Task ARecordedFormatThatDoesNotMatchThePathIsIgnored() { } } + /// + /// A recorded format is never replaced, so it can outlive the file it was recorded for. Raw data on disk are + /// described by what is there, exactly as before formats were recorded: a Waters folder where a Thermo file was + /// recorded is Waters, and a Thermo file where a Waters folder was recorded is Thermo. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task RawDataOnDiskAreDescribedByWhatIsThereWhateverWasRecorded() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var scratch = Path.Combine(Path.GetTempPath(), $"msdial-mztab-replaced-{System.Guid.NewGuid():N}"); + Directory.CreateDirectory(scratch); + try { + Assert.AreEqual(2, storage.AnalysisFiles.Count); + var waters = Path.Combine(scratch, "was_thermo.raw"); + Directory.CreateDirectory(waters); + storage.AnalysisFiles[0].AnalysisFilePath = waters; + storage.AnalysisFiles[0].RawFileFormat = RawFileFormat.ThermoRaw; + var thermo = Path.Combine(scratch, "was_waters.RAW"); + File.WriteAllBytes(thermo, new byte[0]); + storage.AnalysisFiles[1].AnalysisFilePath = thermo; + storage.AnalysisFiles[1].RawFileFormat = RawFileFormat.WatersRaw; + + var formats = MsRunFormats(ExportToLines(storage, container, msdecs)); + + CollectionAssert.AreEqual(new[] { "[MS, MS:1000526, Waters raw format, ]", "[MS, MS:1000563, Thermo RAW format, ]" }, formats); + Assert.AreEqual(RawFileFormat.ThermoRaw, storage.AnalysisFiles[0].RawFileFormat, "exporting does not change what is recorded"); + Assert.AreEqual(RawFileFormat.WatersRaw, storage.AnalysisFiles[1].RawFileFormat, "exporting does not change what is recorded"); + } + finally { + Directory.Delete(scratch, recursive: true); + } + } + /// /// Registration to export: the format is recorded while the raw data are present, the project is saved, the raw /// data are deleted, and the project reloaded from what was saved still names each format with its PSI-MS term. diff --git a/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs b/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs index 6e8b0ef8e..1d967062a 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs @@ -250,6 +250,48 @@ public void ARecordedFormatThatDoesNotFitThePathGivesWayToTheDisk() { Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Resolve(gone)); } + /// + /// A recorded format is never replaced, so it can outlive the file it was recorded for: another file of the same + /// extension may since have taken its place. A file on disk is described by what is there. + /// + [DataTestMethod()] + // content: as in ADotDIsToldByWhatItHolds, at the path itself; "file" for a plain file there. + [DataRow("sample.raw", "", RawFileFormat.ThermoRaw, RawFileFormat.WatersRaw)] + [DataRow("sample.RAW", "file", RawFileFormat.WatersRaw, RawFileFormat.ThermoRaw)] + [DataRow("sample.d", "analysis.tdf", RawFileFormat.BrukerBaf, RawFileFormat.BrukerTdf)] + [DataRow("sample.d", "AcqData/", RawFileFormat.BrukerTdf, RawFileFormat.AgilentMassHunterD)] + [DataRow("SAMPLE.D", "analysis.baf", RawFileFormat.AgilentMassHunterD, RawFileFormat.BrukerBaf)] + public void AFileOnDiskIsDescribedByWhatItIsWhateverWasRecorded(string name, string content, RawFileFormat recorded, RawFileFormat expected) { + var path = Path.Combine(_folder, name); + if (content == "file") { + File.WriteAllBytes(path, new byte[0]); + } + else { + Directory.CreateDirectory(path); + if (content.EndsWith("/")) { + Directory.CreateDirectory(Path.Combine(path, content.TrimEnd('/'))); + } + else if (content.Length > 0) { + File.WriteAllBytes(Path.Combine(path, content), new byte[0]); + } + } + var file = new AnalysisFileBean { AnalysisFilePath = path, RawFileFormat = recorded, }; + + Assert.AreEqual(expected, RawFileFormatClassifier.Resolve(file)); + Assert.AreEqual(recorded, file.RawFileFormat, "resolving does not change what is recorded"); + } + + /// + /// A vendor folder whose marker is gone but which is still there tells nothing, so the recorded format describes it. + /// + [TestMethod()] + public void AFolderThatNoLongerTellsItsFormatIsDescribedByTheRecord() { + var folder = Path.Combine(_folder, "emptied.d"); + Directory.CreateDirectory(folder); + + Assert.AreEqual(RawFileFormat.BrukerTsf, RawFileFormatClassifier.Resolve(new AnalysisFileBean { AnalysisFilePath = folder, RawFileFormat = RawFileFormat.BrukerTsf, })); + } + [TestMethod()] public void NothingRecordedIsToldFromTheDisk() { var path = Path.Combine(_folder, "sample.raw");