diff --git a/src/MSDIAL5/MsdialCore/DataObj/AnalysisFile.cs b/src/MSDIAL5/MsdialCore/DataObj/AnalysisFile.cs index e224e37e4..d64468ab9 100644 --- a/src/MSDIAL5/MsdialCore/DataObj/AnalysisFile.cs +++ b/src/MSDIAL5/MsdialCore/DataObj/AnalysisFile.cs @@ -1,6 +1,7 @@ using CompMs.Common.DataObj; using CompMs.Common.Enum; using CompMs.MsdialCore.Algorithm; +using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.MSDec; using CompMs.MsdialCore.Parameter; using CompMs.MsdialCore.Parser; @@ -62,7 +63,15 @@ public AnalysisFileBean() { public string ProteinAssembledResultFilePath { get; set; } // *.prf [Key(18)] public AcquisitionType AcquisitionType { get; set; } = AcquisitionType.None; - + /// + /// The format of the raw data file, recorded while the file was present (see ), + /// so that it can still be named after the raw data are deleted or moved. when + /// it was never seen, which is also what a project written before this member existed reads as. + /// + /// No initializer: a key missing from an older project is read as default(T), not as an initializer's value. + [Key(19)] + public RawFileFormat RawFileFormat { get; set; } + [IgnoreMember] public bool IsDoMs2ChromDeconvolution => AcquisitionType != AcquisitionType.DDA; diff --git a/src/MSDIAL5/MsdialCore/Enum/RawFileFormat.cs b/src/MSDIAL5/MsdialCore/Enum/RawFileFormat.cs new file mode 100644 index 000000000..ce67e91e6 --- /dev/null +++ b/src/MSDIAL5/MsdialCore/Enum/RawFileFormat.cs @@ -0,0 +1,34 @@ +namespace CompMs.MsdialCore.Enum { + /// + /// What a raw data file is, as far as MS-DIAL can tell from its extension and, for the vendor folders + /// and the two kinds of .raw, from what is on disk. Recorded on + /// when the file is registered, so the format can still be named after the raw data are gone. + /// + /// + /// Persisted by number in projects. Numbers are never reused or renumbered: a new format is appended + /// with the next free number, and stays 0, which is what a project written before + /// the format was recorded reads as. + /// + public enum RawFileFormat + { + Unknown = 0, + ThermoRaw = 1, + WatersRaw = 2, + AgilentMassHunterD = 3, + BrukerTdf = 4, + BrukerTsf = 5, + BrukerBaf = 6, + SciexWiff = 7, + SciexWiff2 = 8, + ShimadzuLcd = 9, + ShimadzuQgd = 10, + MzML = 11, + ImzML = 12, + NetCdf = 13, + Abf = 14, + Ibf = 15, + KanomaxLrp = 16, + HiveHmd = 17, + HiveMzb = 18, + } +} diff --git a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs index 4b2f5994b..dae52cb12 100644 --- a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs +++ b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs @@ -4,8 +4,10 @@ using CompMs.Common.Utility; using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.MSDec; using CompMs.MsdialCore.Parameter; +using CompMs.MsdialCore.Utility; using System; using System.Collections.Generic; using System.IO; @@ -1555,104 +1557,10 @@ private static IReadOnlyDictionary SetRawFileMetadataDic(I var analysisFilePath = files[i].AnalysisFilePath; var analysisFileExtention = Path.GetExtension(analysisFilePath).ToUpper(); - var msRunFormat = ""; // analysed file format - var msRunIDFormat = ""; // analysed file Datapoint Number - switch (analysisFileExtention) - { - case (".ABF"): - msRunFormat = "[,, ABF(Analysis Base File) file, ]"; - msRunIDFormat = "[,, ABF file Datapoint Number, ]"; - break; - case (".IBF"): - msRunFormat = "[,, IBF file, ]"; - msRunIDFormat = "[,, IBF file Datapoint Number, ]"; - break; - case (".WIFF"): - case (".WIFF2"): - msRunFormat = "[MS, MS:1000562, ABI WIFF format, ]"; - msRunIDFormat = "[MS, MS:1000770, WIFF nativeID format, ]"; - break; - case (".D"): - // Agilent and Bruker both write a ".d" folder. Bruker's holds analysis.baf - // (BAF) or analysis.tdf / analysis.tsf (timsTOF); Agilent's holds AcqData. - if (File.Exists(Path.Combine(analysisFilePath, "analysis.tdf"))) - { - msRunFormat = "[MS, MS:1002817, Bruker TDF format, ]"; - msRunIDFormat = "[MS, MS:1002818, Bruker TDF nativeID format, ]"; - } - else if (File.Exists(Path.Combine(analysisFilePath, "analysis.tsf"))) - { - msRunFormat = "[MS, MS:1003282, Bruker TSF format, ]"; - msRunIDFormat = "[MS, MS:1003283, Bruker TSF nativeID format, ]"; - } - else if (File.Exists(Path.Combine(analysisFilePath, "analysis.baf"))) - { - msRunFormat = "[MS, MS:1000815, Bruker BAF format, ]"; - msRunIDFormat = "[MS, MS:1000772, Bruker BAF nativeID format, ]"; - } - else if (Directory.Exists(Path.Combine(analysisFilePath, "AcqData"))) - { - msRunFormat = "[MS, MS:1001509, Agilent MassHunter format, ]"; - msRunIDFormat = "[MS, MS:1001508, Agilent MassHunter nativeID format, ]"; - } - else - { - msRunFormat = "[,, Unknown .d folder format, ]"; - msRunIDFormat = "[,, Unknown file format Datapoint Number, ]"; - } - break; - case (".CDF"): - msRunFormat = "[EDAM, format:3650, netCDF, ]"; - msRunIDFormat = "[MS, MS:1000776, scan number only nativeID format, ]"; - break; - case (".MZML"): - msRunFormat = "[MS, MS:1000584, mzML format, ]"; - msRunIDFormat = "[MS, MS:1000776, scan number only nativeID format, ]"; - break; - case (".RAW"): - // A Waters .raw is a folder and a Thermo .raw a file; a .raw that is not there is neither, - // and is described as unknown, as a .d folder that holds nothing recognisable is. - if (Directory.Exists(analysisFilePath)) - { - msRunFormat = "[MS, MS:1000526, Waters raw format, ]"; - msRunIDFormat = "[MS, MS:1000769, Waters nativeID format, ]"; - } - else if (File.Exists(analysisFilePath)) - { - msRunFormat = "[MS, MS:1000563, Thermo RAW format, ]"; - msRunIDFormat = "[MS, MS:1000768, Thermo nativeID format, ]"; - } - else - { - msRunFormat = "[,, Unknown .raw format, ]"; - msRunIDFormat = "[,, Unknown file format Datapoint Number, ]"; - } - break; - case (".LRP"): - msRunFormat = "[,, LRP file, ]"; - msRunIDFormat = "[,, LRP file Datapoint Number, ]"; - break; - case (".HMD"): - msRunFormat = "[,, Hive HMD file, ]"; - msRunIDFormat = "[,, Hive HMD file Datapoint Number, ]"; - break; - case (".MZB"): - msRunFormat = "[,, Hive mzB file, ]"; - msRunIDFormat = "[,, Hive mzB file Datapoint Number, ]"; - break; - case (".LCD"): - msRunFormat = "[MS, MS:1003009, Shimadzu Biotech LCD format, ]"; - msRunIDFormat = "[MS, MS:1000929, Shimadzu Biotech nativeID format, ]"; - break; - case (".QGD"): - msRunFormat = "[,, Shimadzu GC/MS format, ]"; - msRunIDFormat = "[,, Shimadzu GC/MS format file Datapoint Number, ]"; - break; - default: - msRunFormat = "[,, Unknown file format, ]"; - msRunIDFormat = "[,, Unknown file format Datapoint Number, ]"; - break; - } + // A file on disk is described by what the disk shows, as before formats were recorded. A file whose raw + // data have since been deleted or moved is described by the format recorded while they were present; with + // nothing recorded (a project saved before formats were recorded) it gets the unknown terms. + var (msRunFormat, msRunIDFormat) = MsRunFormatTerms(RawFileFormatClassifier.Resolve(files[i]), analysisFileExtention); msRunIDFormat = "[,, MS-DIAL set Datapoint Number, ]"; var id = files[i].AnalysisFileId + 1; fileMetadataDic.Add(id, new RawFileMetadata() @@ -1676,6 +1584,73 @@ private static IReadOnlyDictionary SetRawFileMetadataDic(I return fileMetadataDic; } + /// + /// The ms_run format and nativeID format terms for a raw data file of . An unknown + /// format is described by a user term that names only the extension (, upper case). + /// + /// + /// These are the terms the exporter wrote when it told the format from the file on disk, kept as they were. + /// The nativeID format is not written: every run's id_format is "MS-DIAL set Datapoint Number", because + /// spectra_ref uses MS-DIAL's scan numbers. imzML has no term here and is written as an unknown file. + /// + internal static (string Format, string IdFormat) MsRunFormatTerms(RawFileFormat format, string upperExtension) + { + switch (format) + { + case RawFileFormat.Abf: + return ("[,, ABF(Analysis Base File) file, ]", "[,, ABF file Datapoint Number, ]"); + case RawFileFormat.Ibf: + return ("[,, IBF file, ]", "[,, IBF file Datapoint Number, ]"); + case RawFileFormat.SciexWiff: + case RawFileFormat.SciexWiff2: + return ("[MS, MS:1000562, ABI WIFF format, ]", "[MS, MS:1000770, WIFF nativeID format, ]"); + // Agilent and Bruker both write a ".d" folder. Bruker's holds analysis.baf + // (BAF) or analysis.tdf / analysis.tsf (timsTOF); Agilent's holds AcqData. + case RawFileFormat.BrukerTdf: + return ("[MS, MS:1002817, Bruker TDF format, ]", "[MS, MS:1002818, Bruker TDF nativeID format, ]"); + case RawFileFormat.BrukerTsf: + return ("[MS, MS:1003282, Bruker TSF format, ]", "[MS, MS:1003283, Bruker TSF nativeID format, ]"); + case RawFileFormat.BrukerBaf: + return ("[MS, MS:1000815, Bruker BAF format, ]", "[MS, MS:1000772, Bruker BAF nativeID format, ]"); + case RawFileFormat.AgilentMassHunterD: + return ("[MS, MS:1001509, Agilent MassHunter format, ]", "[MS, MS:1001508, Agilent MassHunter nativeID format, ]"); + case RawFileFormat.NetCdf: + return ("[EDAM, format:3650, netCDF, ]", "[MS, MS:1000776, scan number only nativeID format, ]"); + case RawFileFormat.MzML: + return ("[MS, MS:1000584, mzML format, ]", "[MS, MS:1000776, scan number only nativeID format, ]"); + // A Waters .raw is a folder and a Thermo .raw a file. + case RawFileFormat.WatersRaw: + return ("[MS, MS:1000526, Waters raw format, ]", "[MS, MS:1000769, Waters nativeID format, ]"); + case RawFileFormat.ThermoRaw: + return ("[MS, MS:1000563, Thermo RAW format, ]", "[MS, MS:1000768, Thermo nativeID format, ]"); + case RawFileFormat.KanomaxLrp: + return ("[,, LRP file, ]", "[,, LRP file Datapoint Number, ]"); + case RawFileFormat.HiveHmd: + return ("[,, Hive HMD file, ]", "[,, Hive HMD file Datapoint Number, ]"); + case RawFileFormat.HiveMzb: + return ("[,, Hive mzB file, ]", "[,, Hive mzB file Datapoint Number, ]"); + case RawFileFormat.ShimadzuLcd: + return ("[MS, MS:1003009, Shimadzu Biotech LCD format, ]", "[MS, MS:1000929, Shimadzu Biotech nativeID format, ]"); + case RawFileFormat.ShimadzuQgd: + return ("[,, Shimadzu GC/MS format, ]", "[,, Shimadzu GC/MS format file Datapoint Number, ]"); + case RawFileFormat.ImzML: + case RawFileFormat.Unknown: + default: + break; + } + // A .raw or .d whose format was never recorded and that is not on disk (or holds nothing recognisable) + // is neither vendor's, and is described as unknown rather than as a vendor it may not be. + switch (upperExtension) + { + case (".RAW"): + return ("[,, Unknown .raw format, ]", "[,, Unknown file format Datapoint Number, ]"); + case (".D"): + return ("[,, Unknown .d folder format, ]", "[,, Unknown file format Datapoint Number, ]"); + default: + return ("[,, Unknown file format, ]", "[,, Unknown file format Datapoint Number, ]"); + } + } + static string SetAdductTypeString(string adductIons) { if (adductIons != "null" && adductIons.Length > 2 && adductIons.Substring(adductIons.Length - 2, 1) == "]") diff --git a/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs b/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs new file mode 100644 index 000000000..5b485a4d9 --- /dev/null +++ b/src/MSDIAL5/MsdialCore/Utility/RawFileFormatClassifier.cs @@ -0,0 +1,163 @@ +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; +using System; +using System.IO; + +namespace CompMs.MsdialCore.Utility +{ + /// + /// Tells the format of a raw data file without reading it, so that it can be recorded on + /// while the file is present. + /// + /// + /// Most formats are told by the extension alone, in any letter case. Two extensions need the disk: a .raw is a + /// Waters folder or a Thermo file, and a .d folder is Agilent's when it holds AcqData and Bruker's when it holds + /// analysis.tdf, analysis.tsf or analysis.baf. A .d is checked in the order RawDataHandler reads it, AcqData + /// first, so the format named is the one whose reader MS-DIAL used. Without the raw data on disk these two are + /// . Nothing here throws: a path that cannot be checked is Unknown. + /// + public static class RawFileFormatClassifier + { + /// + /// The format of the raw data file at , or when it + /// cannot be told. + /// + public static RawFileFormat Classify(string path) { + if (string.IsNullOrEmpty(path)) { + return RawFileFormat.Unknown; + } + try { + switch (Path.GetExtension(path).ToLowerInvariant()) { + case ".raw": return ClassifyDotRaw(path); + case ".d": return ClassifyDotD(path); + case ".wiff": return RawFileFormat.SciexWiff; + case ".wiff2": return RawFileFormat.SciexWiff2; + case ".lcd": return RawFileFormat.ShimadzuLcd; + case ".qgd": return RawFileFormat.ShimadzuQgd; + case ".mzml": return RawFileFormat.MzML; + case ".imzml": return RawFileFormat.ImzML; + case ".cdf": return RawFileFormat.NetCdf; + case ".abf": return RawFileFormat.Abf; + case ".ibf": return RawFileFormat.Ibf; + case ".lrp": return RawFileFormat.KanomaxLrp; + case ".hmd": return RawFileFormat.HiveHmd; + case ".mzb": return RawFileFormat.HiveMzb; + default: return RawFileFormat.Unknown; + } + } + catch (Exception ex) when (IsPathFailure(ex)) { + return RawFileFormat.Unknown; + } + } + + /// + /// Records the format of when none is recorded yet and its raw data tell it. + /// A recorded format is never replaced. Returns whether a format was recorded. + /// + public static bool RecordIfUnknown(AnalysisFileBean file) { + if (file is null || file.RawFileFormat != RawFileFormat.Unknown) { + return false; + } + var format = Classify(file.AnalysisFilePath); + if (format == RawFileFormat.Unknown) { + return false; + } + file.RawFileFormat = format; + return true; + } + + /// + /// Whether a file with (with its leading dot, any letter case) can be of + /// . is consistent with nothing. + /// + public static bool IsConsistentWith(RawFileFormat format, string extension) { + var expected = ExtensionOf(format); + return expected is not null && string.Equals(expected, extension, StringComparison.OrdinalIgnoreCase); + } + + /// + /// The format to describe by: what tells from the disk now, and when + /// the disk cannot tell (the raw data are deleted or moved), the recorded format when it fits the file's extension. + /// + /// + /// The disk comes first because a format is recorded once and never replaced: if another file of the same extension + /// later takes the place of the recorded one (a Waters .raw folder where a Thermo .raw file was), the record no + /// longer describes what is at the path. A file on disk is therefore described by what is there, and the record is + /// used only for what the disk can no longer tell. + /// + public static RawFileFormat Resolve(AnalysisFileBean file) { + var onDisk = Classify(file.AnalysisFilePath); + if (onDisk != RawFileFormat.Unknown) { + return onDisk; + } + var recorded = file.RawFileFormat; + return IsConsistentWith(recorded, ExtensionOrNull(file.AnalysisFilePath)) ? recorded : RawFileFormat.Unknown; + } + + private static RawFileFormat ClassifyDotRaw(string path) { + if (Directory.Exists(path)) { + return RawFileFormat.WatersRaw; + } + if (File.Exists(path)) { + return RawFileFormat.ThermoRaw; + } + return RawFileFormat.Unknown; + } + + private static RawFileFormat ClassifyDotD(string path) { + if (Directory.Exists(Path.Combine(path, "AcqData"))) { + return RawFileFormat.AgilentMassHunterD; + } + if (File.Exists(Path.Combine(path, "analysis.tdf"))) { + return RawFileFormat.BrukerTdf; + } + if (File.Exists(Path.Combine(path, "analysis.tsf"))) { + return RawFileFormat.BrukerTsf; + } + if (File.Exists(Path.Combine(path, "analysis.baf"))) { + return RawFileFormat.BrukerBaf; + } + return RawFileFormat.Unknown; + } + + private static string? ExtensionOf(RawFileFormat format) { + switch (format) { + case RawFileFormat.ThermoRaw: + case RawFileFormat.WatersRaw: + return ".raw"; + case RawFileFormat.AgilentMassHunterD: + case RawFileFormat.BrukerTdf: + case RawFileFormat.BrukerTsf: + case RawFileFormat.BrukerBaf: + return ".d"; + case RawFileFormat.SciexWiff: return ".wiff"; + case RawFileFormat.SciexWiff2: return ".wiff2"; + case RawFileFormat.ShimadzuLcd: return ".lcd"; + case RawFileFormat.ShimadzuQgd: return ".qgd"; + case RawFileFormat.MzML: return ".mzml"; + case RawFileFormat.ImzML: return ".imzml"; + case RawFileFormat.NetCdf: return ".cdf"; + case RawFileFormat.Abf: return ".abf"; + case RawFileFormat.Ibf: return ".ibf"; + case RawFileFormat.KanomaxLrp: return ".lrp"; + case RawFileFormat.HiveHmd: return ".hmd"; + case RawFileFormat.HiveMzb: return ".mzb"; + default: return null; + } + } + + private static string? ExtensionOrNull(string path) { + try { + return Path.GetExtension(path); + } + catch (Exception ex) when (IsPathFailure(ex)) { + return null; + } + } + + private static bool IsPathFailure(Exception ex) { + return ex is ArgumentException || ex is IOException || ex is NotSupportedException + || ex is UnauthorizedAccessException || ex is System.Security.SecurityException; + } + } +} diff --git a/src/MSDIAL5/MsdialGuiApp/Model/DataObj/AnalysisFileBeanModel.cs b/src/MSDIAL5/MsdialGuiApp/Model/DataObj/AnalysisFileBeanModel.cs index f1287d44e..547c0461a 100644 --- a/src/MSDIAL5/MsdialGuiApp/Model/DataObj/AnalysisFileBeanModel.cs +++ b/src/MSDIAL5/MsdialGuiApp/Model/DataObj/AnalysisFileBeanModel.cs @@ -93,6 +93,8 @@ public string AnalysisFilePath { set { if (_file.AnalysisFilePath != value) { _file.AnalysisFilePath = value; + // Another raw data file: what was recorded for the previous one no longer describes it. + _file.RawFileFormat = RawFileFormatClassifier.Classify(value); OnPropertyChanged(nameof(AnalysisFilePath)); _isRawDataAvailable = null; OnPropertyChanged(nameof(IsRawDataAvailable)); @@ -104,14 +106,24 @@ public string AnalysisFilePath { /// Whether the raw data file of this sample is present, as last checked. A runtime fact, never saved: /// the first read checks the disk, and checks again. /// - public bool IsRawDataAvailable => _isRawDataAvailable ??= DataAccess.RawDataExists(_file.AnalysisFilePath); + public bool IsRawDataAvailable => _isRawDataAvailable ??= CheckRawDataExists(); private bool? _isRawDataAvailable; public bool RefreshRawDataAvailability() { - SetRawDataAvailability(DataAccess.RawDataExists(_file.AnalysisFilePath)); + SetRawDataAvailability(CheckRawDataExists()); return _isRawDataAvailable!.Value; } + // Every check of the disk comes here. A project saved before raw data formats were recorded learns the format + // of a file the first time its raw data are found, and keeps it when the project is saved. + private bool CheckRawDataExists() { + var exists = DataAccess.RawDataExists(_file.AnalysisFilePath); + if (exists) { + RawFileFormatClassifier.RecordIfUnknown(_file); + } + return exists; + } + // The folder the raw data file was in is gone, so the file is too; the collection checks a folder once for all its files. internal void MarkRawDataMissing() { SetRawDataAvailability(false); diff --git a/src/MSDIAL5/MsdialGuiApp/Model/Setting/DatasetFileSettingModel.cs b/src/MSDIAL5/MsdialGuiApp/Model/Setting/DatasetFileSettingModel.cs index d60b44ec9..083e1f6af 100644 --- a/src/MSDIAL5/MsdialGuiApp/Model/Setting/DatasetFileSettingModel.cs +++ b/src/MSDIAL5/MsdialGuiApp/Model/Setting/DatasetFileSettingModel.cs @@ -6,6 +6,7 @@ using CompMs.MsdialCore.DataObj; using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.Parameter; +using CompMs.MsdialCore.Utility; using System; using System.Collections.Generic; using System.IO; @@ -81,6 +82,8 @@ public void SetFiles(IEnumerable files) { ProteinAssembledResultFilePath = Path.Combine(folder, $"{name}_{_dateTime:yyyyMMddHHmm}.{MsdialDataStorageFormat.prf}"), RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean(Path.Combine(folder, $"{name}_{_dateTime:yyyyMMddHHmm}.{MsdialDataStorageFormat.rtc}")), AcquisitionType = AcquisitionType.DDA, + // Recorded while the file is present, so mzTab-M can still name its format after the raw data are gone. + RawFileFormat = RawFileFormatClassifier.Classify(file), }; FileModels.AddAnalysisFile(bean); checker.CheckFileType(bean.AnalysisFilePath); diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/AnalysisFilesParser.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/AnalysisFilesParser.cs index ba21b34e7..d1b0c4b09 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/AnalysisFilesParser.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/AnalysisFilesParser.cs @@ -237,7 +237,9 @@ public static List ReadCsvContents(string filepath) PeakAreaBeanInformationFilePath = Path.Combine(fileDir, afFilename + "_" + dtString + ".pai"), RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean(fileDir + "\\" + afFilename + "_" + dtString + ".rtc"), AnalysisBatch = afBatch, - DilutionFactor = afInjectVolume + DilutionFactor = afInjectVolume, + // Recorded while the file is present, so mzTab-M can still name its format after the raw data are gone. + RawFileFormat = RawFileFormatClassifier.Classify(afFilepath), }); counter++; } @@ -286,7 +288,9 @@ public static List CreateAnalysisFileBeans(List filePa AnalysisFileType = AnalysisFileType.Sample, DeconvolutionFilePath = Path.Combine(fileDir, filename + "_" + dtString + ".dcl"), PeakAreaBeanInformationFilePath = Path.Combine(fileDir, filename + "_" + dtString + ".pai"), - RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean(Path.Combine(fileDir, filename + "_" + dtString + ".rtc")) + RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean(Path.Combine(fileDir, filename + "_" + dtString + ".rtc")), + // Recorded while the file is present, so mzTab-M can still name its format after the raw data are gone. + RawFileFormat = RawFileFormatClassifier.Classify(filepath), }); counter++; } diff --git a/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/AnalysisFilesParserRawFileFormatTests.cs b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/AnalysisFilesParserRawFileFormatTests.cs new file mode 100644 index 000000000..79c71f214 --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/AnalysisFilesParserRawFileFormatTests.cs @@ -0,0 +1,111 @@ +using CompMs.App.MsdialConsole.Parser; +using CompMs.MsdialCore.Enum; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; + +namespace MsdialCoreTestAppTests.Parser; + +/// +/// The Console records each raw data file's format when it reads the analysis files, while the files are surely there, +/// so the format survives in the project after the raw data are deleted. +/// +[TestClass] +public sealed class AnalysisFilesParserRawFileFormatTests +{ + [TestMethod] + public void ReadFolderContents_RecordsEachFormat() + { + using var directory = VendorFolder(); + + var files = AnalysisFilesParser.ReadFolderContents(directory.Path); + + AssertFormats(files.Select(file => (Path.GetFileName(file.AnalysisFilePath), file.RawFileFormat))); + } + + [TestMethod] + public void ReadCsvContents_RecordsEachFormat() + { + using var directory = VendorFolder(); + var names = new[] { "thermo.raw", "waters.raw", "agilent.d", "bruker.d", "sample.mzML", "sample.lcd" }; + var csv = directory.CreateFile( + "analysis_files.csv", + "file_path,file_name,file_type,class_id,acquisition_type,batch_order,analytical_order,factor" + Environment.NewLine + + string.Join(Environment.NewLine, names.Select((name, i) => $"{System.IO.Path.Combine(directory.Path, name)},{Path.GetFileNameWithoutExtension(name)}_{i},Sample,A,DDA,1,{i + 1},1"))); + + var files = AnalysisFilesParser.ReadCsvContents(csv); + + Assert.AreEqual(names.Length, files.Count); + AssertFormats(files.Select(file => (Path.GetFileName(file.AnalysisFilePath), file.RawFileFormat))); + } + + [TestMethod] + public void ReadInput_ASingleFile_RecordsItsFormat() + { + using var directory = new TemporaryDirectory(); + var path = directory.CreateFile("thermo.raw"); + + var files = AnalysisFilesParser.ReadInput(path, new StringReader(string.Empty), new StringWriter(), inputRedirected: true, outputRedirected: true); + + Assert.IsNotNull(files); + Assert.AreEqual(RawFileFormat.ThermoRaw, files.Single().RawFileFormat); + } + + private static void AssertFormats(System.Collections.Generic.IEnumerable<(string Name, RawFileFormat Format)> actual) + { + var formats = actual.ToDictionary(pair => pair.Name, pair => pair.Format); + Assert.AreEqual(RawFileFormat.ThermoRaw, formats["thermo.raw"]); + Assert.AreEqual(RawFileFormat.WatersRaw, formats["waters.raw"]); + Assert.AreEqual(RawFileFormat.AgilentMassHunterD, formats["agilent.d"]); + Assert.AreEqual(RawFileFormat.BrukerTdf, formats["bruker.d"]); + Assert.AreEqual(RawFileFormat.MzML, formats["sample.mzML"]); + Assert.AreEqual(RawFileFormat.ShimadzuLcd, formats["sample.lcd"]); + } + + private static TemporaryDirectory VendorFolder() + { + var directory = new TemporaryDirectory(); + directory.CreateFile("thermo.raw"); + directory.CreateDirectory("waters.raw"); + directory.CreateDirectory(System.IO.Path.Combine("agilent.d", "AcqData")); + directory.CreateDirectory("bruker.d"); + directory.CreateFile(System.IO.Path.Combine("bruker.d", "analysis.tdf")); + directory.CreateFile("sample.mzML"); + directory.CreateFile("sample.lcd"); + return directory; + } + + private sealed class TemporaryDirectory : IDisposable + { + public TemporaryDirectory() + { + Path = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + "MsdialCoreTestAppTests", + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public string CreateDirectory(string name) + { + var path = System.IO.Path.Combine(Path, name); + Directory.CreateDirectory(path); + return path; + } + + public string CreateFile(string name, string content = "") + { + var path = System.IO.Path.Combine(Path, name); + File.WriteAllText(path, content); + return path; + } + + public void Dispose() + { + Directory.Delete(Path, recursive: true); + } + } +} diff --git a/tests/MSDIAL5/MsdialCoreTests/DataObj/AnalysisFileBeanRawFileFormatTests.cs b/tests/MSDIAL5/MsdialCoreTests/DataObj/AnalysisFileBeanRawFileFormatTests.cs new file mode 100644 index 000000000..2f227be0a --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTests/DataObj/AnalysisFileBeanRawFileFormatTests.cs @@ -0,0 +1,258 @@ +using CompMs.Common.Enum; +using CompMs.Common.MessagePack; +using CompMs.MsdialCore.Enum; +using MessagePack; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace CompMs.MsdialCore.DataObj.Tests +{ + /// + /// The raw data format is recorded as MessagePack key 19 of . Projects written before + /// it must still load and read as , and an MS-DIAL that knows only keys 0-18 + /// (5.5, and the 5.6 line at the time of writing) must still open a project that has it. + /// + /// + /// Project IO goes through (LZ4 over the stock resolver), where an int-keyed + /// [MessagePackObject] is an array indexed by key: a shorter array is read to its own length, and the elements past + /// the keys a reader knows are skipped. These tests pin both directions for this type, as + /// does for match results. + /// + [TestClass()] + public class AnalysisFileBeanRawFileFormatTests + { + [TestMethod()] + public void EveryFormatSurvivesSavingAndLoading() { + foreach (var format in AllFormats()) { + var bean = new AnalysisFileBean { AnalysisFileName = "sample", RawFileFormat = format, }; + + var actual = RoundTrip(bean); + + Assert.AreEqual(format, actual.RawFileFormat); + Assert.AreEqual("sample", actual.AnalysisFileName); + } + } + + [TestMethod()] + public void ANewBeanRecordsNothing() { + Assert.AreEqual(RawFileFormat.Unknown, new AnalysisFileBean().RawFileFormat); + } + + /// + /// A dataset saved on 2025-07-31, before key 19 existed: every file reads as Unknown, and keys 0-18 read exactly as + /// an MS-DIAL without key 19 reads them. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + public void AProjectWrittenBeforeTheFormatWasRecordedReadsAsUnknown() { + var bytes = File.ReadAllBytes(Path.Combine("Resources", "Export", "Dataset_2025_07_31_12_31_11.mddata")); + + var storage = LZ4MessagePackSerializer.Deserialize(bytes); + + Assert.IsTrue(storage.AnalysisFiles.Count > 0); + Assert.IsTrue(storage.AnalysisFiles.All(file => file.RawFileFormat == RawFileFormat.Unknown)); + + var beans = AnalysisFileBlocks(LZ4MessagePackSerializer.Decode(bytes)).ToList(); + Assert.AreEqual(storage.AnalysisFiles.Count, beans.Count); + foreach (var (block, actual) in beans.Zip(storage.AnalysisFiles, (block, actual) => (block, actual))) { + Assert.AreEqual(19, MessagePackBinary.ReadArrayHeader(block, 0, out _), "the fixture must predate key 19"); + var old = MessagePackSerializer.Deserialize(block); + AssertSameKeys0To18(old, actual); + } + } + + /// + /// The other direction: an MS-DIAL that knows keys 0-18 opens a project written with key 19. The beans sit in a list + /// with more data after it, so a reader that skipped key 19 by the wrong length would misread what follows. + /// + [TestMethod()] + public void AnOlderMsdialReadsAProjectWithTheFormatRecorded() { + var written = new BeansThenMore + { + AnalysisFiles = new List { CreateFullBean(0, RawFileFormat.BrukerTdf), CreateFullBean(1, RawFileFormat.HiveMzb), }, + After = "after the beans", + }; + var stream = new MemoryStream(); + MessagePackDefaultHandler.SaveToStream(written, stream); + + stream.Position = 0; + var read = MessagePackDefaultHandler.LoadFromStream(stream); + + Assert.AreEqual(2, read.AnalysisFiles.Count); + Assert.AreEqual("after the beans", read.After); + for (int i = 0; i < 2; i++) { + AssertSameKeys0To18(read.AnalysisFiles[i], written.AnalysisFiles[i]); + } + } + + [TestMethod()] + public void AnOlderMsdialReadsASingleBeanWithTheFormatRecorded() { + var bean = CreateFullBean(3, RawFileFormat.WatersRaw); + var bytes = MessagePackSerializer.Serialize(bean); + Assert.AreEqual(20, MessagePackBinary.ReadArrayHeader(bytes, 0, out _), "key 19 is written"); + + var old = MessagePackSerializer.Deserialize(bytes); + + AssertSameKeys0To18(old, bean); + } + + /// + /// Projects store the number, so a number once given to a format is never given to another. + /// New formats are appended. + /// + [TestMethod()] + public void TheFormatNumbersAreFixed() { + var expected = new Dictionary + { + [RawFileFormat.Unknown] = 0, + [RawFileFormat.ThermoRaw] = 1, + [RawFileFormat.WatersRaw] = 2, + [RawFileFormat.AgilentMassHunterD] = 3, + [RawFileFormat.BrukerTdf] = 4, + [RawFileFormat.BrukerTsf] = 5, + [RawFileFormat.BrukerBaf] = 6, + [RawFileFormat.SciexWiff] = 7, + [RawFileFormat.SciexWiff2] = 8, + [RawFileFormat.ShimadzuLcd] = 9, + [RawFileFormat.ShimadzuQgd] = 10, + [RawFileFormat.MzML] = 11, + [RawFileFormat.ImzML] = 12, + [RawFileFormat.NetCdf] = 13, + [RawFileFormat.Abf] = 14, + [RawFileFormat.Ibf] = 15, + [RawFileFormat.KanomaxLrp] = 16, + [RawFileFormat.HiveHmd] = 17, + [RawFileFormat.HiveMzb] = 18, + }; + foreach (var pair in expected) { + Assert.AreEqual(pair.Value, (int)pair.Key, pair.Key.ToString()); + } + // A value added at the end needs a line above; one added anywhere else breaks a line above. + CollectionAssert.AreEquivalent(expected.Keys.ToArray(), AllFormats().ToArray()); + } + + private static IEnumerable AllFormats() { + return System.Enum.GetValues(typeof(RawFileFormat)).Cast(); + } + + private static AnalysisFileBean RoundTrip(AnalysisFileBean bean) { + using var stream = new MemoryStream(); + MessagePackDefaultHandler.SaveToStream(bean, stream); + stream.Position = 0; + return MessagePackDefaultHandler.LoadFromStream(stream); + } + + /// + /// The MessagePack block of each analysis file in a serialized , whose key 0 is the + /// list of analysis files. + /// + private static IEnumerable AnalysisFileBlocks(byte[] storage) { + var offset = 0; + MessagePackBinary.ReadArrayHeader(storage, offset, out var readSize); + offset += readSize; + var count = MessagePackBinary.ReadArrayHeader(storage, offset, out readSize); + offset += readSize; + for (int i = 0; i < count; i++) { + var size = MessagePackBinary.ReadNextBlock(storage, offset); + var block = new byte[size]; + Array.Copy(storage, offset, block, 0, size); + offset += size; + yield return block; + } + } + + private static AnalysisFileBean CreateFullBean(int id, RawFileFormat format) { + return new AnalysisFileBean + { + AnalysisFilePath = $@"D:\data\sample_{id}.d", + AnalysisFileName = $"sample_{id}", + AnalysisFileType = AnalysisFileType.QC, + AnalysisFileClass = "class A", + AnalysisFileAnalyticalOrder = 10 + id, + AnalysisFileId = id, + AnalysisFileIncluded = true, + DeconvolutionFilePath = $@"D:\data\sample_{id}.dcl", + DeconvolutionFilePathList = new List { $@"D:\data\sample_{id}_1000.dcl", $@"D:\data\sample_{id}_2000.dcl", }, + PeakAreaBeanInformationFilePath = $@"D:\data\sample_{id}.pai", + RiDictionaryFilePath = $@"D:\data\ri_{id}.txt", + AnalysisBatch = 2, + ResponseVariable = 0.25, + DilutionFactor = 4d, + AnalysisFileSuperClass = "super", + RetentionTimeCorrectionBean = new RetentionTimeCorrectionBean($@"D:\data\sample_{id}.rtc"), + ProteinAssembledResultFilePath = $@"D:\data\sample_{id}.prf", + AcquisitionType = AcquisitionType.SWATH, + RawFileFormat = format, + }; + } + + /// + /// Keys 0-18 must be read the same by both. Each key is compared by its MessagePack encoding, so nested objects + /// (keys 15 and 16) are compared whole. + /// + private static void AssertSameKeys0To18(AnalysisFileBeanWithoutRawFileFormat old, AnalysisFileBean actual) { + var oldBytes = MessagePackSerializer.Serialize(old); + var actualBytes = MessagePackSerializer.Serialize(actual); + Assert.AreEqual(19, MessagePackBinary.ReadArrayHeader(oldBytes, 0, out var oldOffset)); + Assert.AreEqual(20, MessagePackBinary.ReadArrayHeader(actualBytes, 0, out var actualOffset)); + for (int key = 0; key < 19; key++) { + var oldSize = MessagePackBinary.ReadNextBlock(oldBytes, oldOffset); + var actualSize = MessagePackBinary.ReadNextBlock(actualBytes, actualOffset); + CollectionAssert.AreEqual( + new ArraySegment(oldBytes, oldOffset, oldSize).ToArray(), + new ArraySegment(actualBytes, actualOffset, actualSize).ToArray(), + $"key {key}"); + oldOffset += oldSize; + actualOffset += actualSize; + } + Assert.AreEqual(old.AnalysisFilePath, actual.AnalysisFilePath); + Assert.AreEqual(old.AnalysisFileName, actual.AnalysisFileName); + Assert.AreEqual(old.AnalysisFileId, actual.AnalysisFileId); + Assert.AreEqual(old.AcquisitionType, actual.AcquisitionType); + } + + /// + /// as an MS-DIAL without key 19 declares it. + /// + [MessagePackObject] + public class AnalysisFileBeanWithoutRawFileFormat + { + [Key(0)] public string AnalysisFilePath { get; set; } = string.Empty; + [Key(1)] public string AnalysisFileName { get; set; } = string.Empty; + [Key(2)] public AnalysisFileType AnalysisFileType { get; set; } + [Key(3)] public string AnalysisFileClass { get; set; } = string.Empty; + [Key(4)] public int AnalysisFileAnalyticalOrder { get; set; } + [Key(5)] public int AnalysisFileId { get; set; } + [Key(6)] public bool AnalysisFileIncluded { get; set; } + [Key(7)] public string DeconvolutionFilePath { get; set; } = string.Empty; + [Key(8)] public List DeconvolutionFilePathList { get; set; } = new List(); + [Key(9)] public string PeakAreaBeanInformationFilePath { get; set; } = string.Empty; + [Key(10)] public string RiDictionaryFilePath { get; set; } = string.Empty; + [Key(11)] public int AnalysisBatch { get; set; } = 1; + [Key(12)] public double ResponseVariable { get; set; } = 0; + [Key(13)] public double DilutionFactor { get; set; } = 1.0; + [Key(14)] public string AnalysisFileSuperClass { get; set; } = string.Empty; + [Key(15)] public RetentionTimeCorrectionBean RetentionTimeCorrectionBean { get; set; } = new RetentionTimeCorrectionBean(); + [Key(16)] public ChromatogramPeaksDataSummaryDto ChromPeakFeaturesSummary { get; set; } = new ChromatogramPeaksDataSummaryDto(); + [Key(17)] public string ProteinAssembledResultFilePath { get; set; } + [Key(18)] public AcquisitionType AcquisitionType { get; set; } = AcquisitionType.None; + } + + [MessagePackObject] + public class BeansThenMore + { + [Key(0)] public List AnalysisFiles { get; set; } = new List(); + [Key(1)] public string After { get; set; } = string.Empty; + } + + [MessagePackObject] + public class OldBeansThenMore + { + [Key(0)] public List AnalysisFiles { get; set; } = new List(); + [Key(1)] public string After { get; set; } = string.Empty; + } + } +} diff --git a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs index 6a848ea43..d4c173e5b 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs @@ -4,8 +4,10 @@ using CompMs.Common.Utility; using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; using CompMs.MsdialCore.Parameter; using CompMs.MsdialCore.Parser; +using CompMs.MsdialCore.Utility; using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Collections.Generic; using System.IO; @@ -173,8 +175,8 @@ private static string[] ExportToLines( /// /// Agilent and Bruker both write a ".d" folder, and ms_run-format is told apart by what the folder holds. - /// A folder that is not on disk when the file is exported (moved, archived, or a project opened on another - /// machine) gets a neutral user term, not a vendor it may not be. + /// When no format was recorded for it, a folder that is not on disk when the file is exported (moved, archived, + /// or a project opened on another machine) gets a neutral user term, not a vendor it may not be. /// /// /// ms_run-id_format is not checked: SetRawFileMetadataDic overwrites it with "MS-DIAL set Datapoint Number" @@ -229,8 +231,9 @@ public async Task ADotDFolderIsDescribedByWhatItHolds(string content, string exp } /// - /// A .raw is a Waters folder or a Thermo file. One that is not on disk when the file is exported used to throw - /// from File.GetAttributes; it gets a neutral user term, as an unrecognisable .d folder does. + /// A .raw is a Waters folder or a Thermo file. When no format was recorded for it, one that is not on disk when + /// the file is exported (which used to throw from File.GetAttributes) gets a neutral user term, as an + /// unrecognisable .d folder does. /// [DataTestMethod()] [DataRow("folder", "[MS, MS:1000526, Waters raw format, ]")] @@ -272,6 +275,288 @@ public async Task ADotRawPathIsDescribedByWhatItIs(string kind, string expectedF } } + /// + /// A .d folder is read as RawDataHandler reads it, AcqData first, so one that holds both an AcqData folder and a + /// Bruker analysis file is Agilent's. This is the one case whose ms_run-format changed when formats began to be + /// recorded: it used to be described by the Bruker file. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task ADotDFolderHoldingAcqDataIsAgilentsWhateverElseItHolds() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var scratch = Path.Combine(Path.GetTempPath(), $"msdial-mztab-dotd-both-{System.Guid.NewGuid():N}"); + try { + foreach (var file in storage.AnalysisFiles) { + var folder = Path.Combine(scratch, file.AnalysisFileName + ".d"); + Directory.CreateDirectory(Path.Combine(folder, "AcqData")); + File.WriteAllBytes(Path.Combine(folder, "analysis.tdf"), new byte[0]); + file.AnalysisFilePath = folder; + } + + var formats = MsRunFormats(ExportToLines(storage, container, msdecs)); + + Assert.AreEqual(storage.AnalysisFiles.Count, formats.Length); + Assert.IsTrue(formats.All(format => format == "[MS, MS:1001509, Agilent MassHunter format, ]"), string.Join(" | ", formats)); + } + finally { + Directory.Delete(scratch, recursive: true); + } + } + + /// + /// A format recorded while the raw data were present is what ms_run-format says after they are gone, with the + /// term the exporter writes for that format on disk. + /// + [DataTestMethod()] + [DataRow(RawFileFormat.ThermoRaw, ".raw", "[MS, MS:1000563, Thermo RAW format, ]")] + [DataRow(RawFileFormat.WatersRaw, ".raw", "[MS, MS:1000526, Waters raw format, ]")] + [DataRow(RawFileFormat.AgilentMassHunterD, ".d", "[MS, MS:1001509, Agilent MassHunter format, ]")] + [DataRow(RawFileFormat.BrukerTdf, ".d", "[MS, MS:1002817, Bruker TDF format, ]")] + [DataRow(RawFileFormat.BrukerTsf, ".d", "[MS, MS:1003282, Bruker TSF format, ]")] + [DataRow(RawFileFormat.BrukerBaf, ".d", "[MS, MS:1000815, Bruker BAF format, ]")] + [DataRow(RawFileFormat.SciexWiff, ".wiff", "[MS, MS:1000562, ABI WIFF format, ]")] + [DataRow(RawFileFormat.ShimadzuLcd, ".lcd", "[MS, MS:1003009, Shimadzu Biotech LCD format, ]")] + [DataRow(RawFileFormat.MzML, ".mzML", "[MS, MS:1000584, mzML format, ]")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task ARecordedFormatIsWrittenWhenTheRawDataAreGone(RawFileFormat recorded, string extension, string expectedFormat) { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var gone = Path.Combine(Path.GetTempPath(), $"msdial-mztab-gone-{System.Guid.NewGuid():N}"); + foreach (var file in storage.AnalysisFiles) { + file.AnalysisFilePath = Path.Combine(gone, file.AnalysisFileName + extension); + file.RawFileFormat = recorded; + } + + var formats = MsRunFormats(ExportToLines(storage, container, msdecs)); + + Assert.AreEqual(storage.AnalysisFiles.Count, formats.Length); + Assert.IsTrue(formats.All(format => format == expectedFormat), string.Join(" | ", formats)); + foreach (var format in formats) { + AssertIsAPsiMsFileFormat(format); + } + Assert.IsFalse(Directory.Exists(gone), "exporting must not create the raw data folder"); + } + + /// + /// A recorded format that cannot be the file at the path (a Thermo format on a .d) is not trusted; the file is + /// described by what the disk shows, as when nothing is recorded. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task ARecordedFormatThatDoesNotMatchThePathIsIgnored() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var scratch = Path.Combine(Path.GetTempPath(), $"msdial-mztab-mismatch-{System.Guid.NewGuid():N}"); + Directory.CreateDirectory(scratch); + try { + Assert.AreEqual(2, storage.AnalysisFiles.Count); + // Present: a Bruker TDF folder recorded as Thermo. Gone: a .d recorded as Thermo. + var present = Path.Combine(scratch, "present.d"); + Directory.CreateDirectory(present); + File.WriteAllBytes(Path.Combine(present, "analysis.tdf"), new byte[0]); + storage.AnalysisFiles[0].AnalysisFilePath = present; + storage.AnalysisFiles[0].RawFileFormat = RawFileFormat.ThermoRaw; + storage.AnalysisFiles[1].AnalysisFilePath = Path.Combine(scratch, "gone.d"); + storage.AnalysisFiles[1].RawFileFormat = RawFileFormat.ThermoRaw; + + var formats = MsRunFormats(ExportToLines(storage, container, msdecs)); + + CollectionAssert.AreEqual(new[] { "[MS, MS:1002817, Bruker TDF format, ]", "[,, Unknown .d folder format, ]" }, formats); + } + finally { + Directory.Delete(scratch, recursive: true); + } + } + + /// + /// A recorded format is never replaced, so it can outlive the file it was recorded for. Raw data on disk are + /// described by what is there, exactly as before formats were recorded: a Waters folder where a Thermo file was + /// recorded is Waters, and a Thermo file where a Waters folder was recorded is Thermo. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task RawDataOnDiskAreDescribedByWhatIsThereWhateverWasRecorded() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var scratch = Path.Combine(Path.GetTempPath(), $"msdial-mztab-replaced-{System.Guid.NewGuid():N}"); + Directory.CreateDirectory(scratch); + try { + Assert.AreEqual(2, storage.AnalysisFiles.Count); + var waters = Path.Combine(scratch, "was_thermo.raw"); + Directory.CreateDirectory(waters); + storage.AnalysisFiles[0].AnalysisFilePath = waters; + storage.AnalysisFiles[0].RawFileFormat = RawFileFormat.ThermoRaw; + var thermo = Path.Combine(scratch, "was_waters.RAW"); + File.WriteAllBytes(thermo, new byte[0]); + storage.AnalysisFiles[1].AnalysisFilePath = thermo; + storage.AnalysisFiles[1].RawFileFormat = RawFileFormat.WatersRaw; + + var formats = MsRunFormats(ExportToLines(storage, container, msdecs)); + + CollectionAssert.AreEqual(new[] { "[MS, MS:1000526, Waters raw format, ]", "[MS, MS:1000563, Thermo RAW format, ]" }, formats); + Assert.AreEqual(RawFileFormat.ThermoRaw, storage.AnalysisFiles[0].RawFileFormat, "exporting does not change what is recorded"); + Assert.AreEqual(RawFileFormat.WatersRaw, storage.AnalysisFiles[1].RawFileFormat, "exporting does not change what is recorded"); + } + finally { + Directory.Delete(scratch, recursive: true); + } + } + + /// + /// Registration to export: the format is recorded while the raw data are present, the project is saved, the raw + /// data are deleted, and the project reloaded from what was saved still names each format with its PSI-MS term. + /// + [DataTestMethod()] + [DataRow("thermo", "waters")] + [DataRow("bruker-tdf", "agilent")] + [DataRow("bruker-tsf", "bruker-baf")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task AFormatRecordedAtRegistrationOutlivesTheRawData(string firstKind, string secondKind) { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + Assert.AreEqual(2, storage.AnalysisFiles.Count); + var kinds = new[] { firstKind, secondKind }; + var scratch = Path.Combine(Path.GetTempPath(), $"msdial-mztab-e2e-{System.Guid.NewGuid():N}"); + var expected = new string[kinds.Length]; + try { + for (int i = 0; i < kinds.Length; i++) { + var file = storage.AnalysisFiles[i]; + (file.AnalysisFilePath, expected[i]) = CreateRawData(scratch, file.AnalysisFileName, kinds[i]); + Assert.AreEqual(RawFileFormat.Unknown, file.RawFileFormat, "the fixture predates recorded formats"); + Assert.IsTrue(RawFileFormatClassifier.RecordIfUnknown(file), kinds[i]); + } + } + finally { + // The project is saved and exported only after the raw data are gone. + Directory.Delete(scratch, recursive: true); + } + + var memory = new MemoryStream(); + using (IStreamManager manager = ZipStreamManager.OpenCreate(memory)) { + await storage.SaveAsync(manager, "Test", "TestFolder"); + manager.Complete(); + } + IMsdialDataStorage reloaded; + using (var manager = ZipStreamManager.OpenGet(memory)) { + reloaded = await MsdialDataStorage.Serializer.LoadAsync(manager, "Test", null, "TestFolder"); + } + + var formats = MsRunFormats(ExportToLines(reloaded, container, msdecs)); + + CollectionAssert.AreEqual(expected, formats); + foreach (var format in formats) { + AssertIsAPsiMsFileFormat(format); + } + Assert.IsFalse(Directory.Exists(scratch), "exporting must not create the raw data folder"); + } + + /// + /// Every PSI-MS term the exporter can write as ms_run-format, for any recorded format, is a file format term. + /// + [TestMethod()] + public void EveryPsiMsFormatTermIsAFileFormat() { + var written = 0; + foreach (var format in System.Enum.GetValues(typeof(RawFileFormat)).Cast()) { + foreach (var extension in new[] { ".RAW", ".D", ".MZML", ".TXT" }) { + var term = MztabFormatExporter.MsRunFormatTerms(format, extension).Format; + if (term.StartsWith("[MS,")) { + AssertIsAPsiMsFileFormat(term); + written++; + } + } + } + Assert.IsTrue(written > 0); + } + + /// + /// The children of MS:1000560 (mass spectrometer file format) that the exporter writes, with their names as in + /// psi-ms.obo (checked against data-version 4.2.2, where none of them is obsolete). + /// + private static readonly Dictionary PsiMsFileFormats = new Dictionary + { + ["MS:1000526"] = "Waters raw format", + ["MS:1000562"] = "ABI WIFF format", + ["MS:1000563"] = "Thermo RAW format", + ["MS:1000584"] = "mzML format", + ["MS:1000815"] = "Bruker BAF format", + ["MS:1001509"] = "Agilent MassHunter format", + ["MS:1002817"] = "Bruker TDF format", + ["MS:1003009"] = "Shimadzu Biotech LCD format", + ["MS:1003282"] = "Bruker TSF format", + }; + + private static void AssertIsAPsiMsFileFormat(string term) { + // An mzTab-M parameter: [CV label, accession, name, value]. + var fields = term.TrimStart('[').TrimEnd(']').Split(',').Select(field => field.Trim()).ToArray(); + Assert.AreEqual(4, fields.Length, term); + Assert.AreEqual("MS", fields[0], term); + Assert.IsTrue(PsiMsFileFormats.TryGetValue(fields[1], out var name), $"{fields[1]} is not a known file format term: {term}"); + Assert.AreEqual(name, fields[2], term); + Assert.AreEqual(string.Empty, fields[3], term); + } + + private static (string Path, string ExpectedFormat) CreateRawData(string folder, string name, string kind) { + switch (kind) { + case "thermo": { + var path = Path.Combine(folder, name + ".raw"); + Directory.CreateDirectory(folder); + File.WriteAllBytes(path, new byte[0]); + return (path, "[MS, MS:1000563, Thermo RAW format, ]"); + } + case "waters": { + var path = Path.Combine(folder, name + ".raw"); + Directory.CreateDirectory(path); + return (path, "[MS, MS:1000526, Waters raw format, ]"); + } + case "agilent": { + var path = Path.Combine(folder, name + ".d"); + Directory.CreateDirectory(Path.Combine(path, "AcqData")); + return (path, "[MS, MS:1001509, Agilent MassHunter format, ]"); + } + case "bruker-tdf": + case "bruker-tsf": + case "bruker-baf": { + var path = Path.Combine(folder, name + ".d"); + Directory.CreateDirectory(path); + File.WriteAllBytes(Path.Combine(path, "analysis." + kind.Substring("bruker-".Length)), new byte[0]); + var expected = kind == "bruker-tdf" ? "[MS, MS:1002817, Bruker TDF format, ]" + : kind == "bruker-tsf" ? "[MS, MS:1003282, Bruker TSF format, ]" + : "[MS, MS:1000815, Bruker BAF format, ]"; + return (path, expected); + } + default: + throw new System.ArgumentException(kind); + } + } + + private static string[] MsRunFormats(string[] lines) { + return lines.Select(line => line.Split('\t')) + .Where(f => f[0] == "MTD" && f.Length > 2 && f[1].StartsWith("ms_run[") && f[1].EndsWith("]-format")) + .Select(f => f[2]) + .ToArray(); + } + [TestMethod()] //[DeploymentItem(@"Resources\Export\small_test_project.mdproject", @"Resources\Export")] diff --git a/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs b/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs new file mode 100644 index 000000000..1d967062a --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTests/Utility/RawFileFormatClassifierTests.cs @@ -0,0 +1,303 @@ +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; + +namespace CompMs.MsdialCore.Utility.Tests; + +[TestClass()] +public class RawFileFormatClassifierTests +{ + private string _folder = string.Empty; + + [TestInitialize()] + public void Setup() { + _folder = Path.Combine(Path.GetTempPath(), $"msdial-rawfileformat-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_folder); + } + + [TestCleanup()] + public void Cleanup() { + Directory.Delete(_folder, recursive: true); + } + + /// + /// These formats are told by the extension alone, in any letter case, so the file need not be there. + /// + [DataTestMethod()] + [DataRow("sample.abf", RawFileFormat.Abf)] + [DataRow("sample.ABF", RawFileFormat.Abf)] + [DataRow("sample.ibf", RawFileFormat.Ibf)] + [DataRow("sample.IBF", RawFileFormat.Ibf)] + [DataRow("sample.wiff", RawFileFormat.SciexWiff)] + [DataRow("sample.WIFF", RawFileFormat.SciexWiff)] + [DataRow("sample.wiff2", RawFileFormat.SciexWiff2)] + [DataRow("sample.WIFF2", RawFileFormat.SciexWiff2)] + [DataRow("sample.lcd", RawFileFormat.ShimadzuLcd)] + [DataRow("sample.LCD", RawFileFormat.ShimadzuLcd)] + [DataRow("sample.qgd", RawFileFormat.ShimadzuQgd)] + [DataRow("sample.QGD", RawFileFormat.ShimadzuQgd)] + [DataRow("sample.mzML", RawFileFormat.MzML)] + [DataRow("sample.mzml", RawFileFormat.MzML)] + [DataRow("sample.MZML", RawFileFormat.MzML)] + [DataRow("sample.imzML", RawFileFormat.ImzML)] + [DataRow("sample.IMZML", RawFileFormat.ImzML)] + [DataRow("sample.cdf", RawFileFormat.NetCdf)] + [DataRow("sample.CDF", RawFileFormat.NetCdf)] + [DataRow("sample.lrp", RawFileFormat.KanomaxLrp)] + [DataRow("sample.LRP", RawFileFormat.KanomaxLrp)] + [DataRow("sample.hmd", RawFileFormat.HiveHmd)] + [DataRow("sample.HMD", RawFileFormat.HiveHmd)] + [DataRow("sample.mzb", RawFileFormat.HiveMzb)] + [DataRow("sample.MZB", RawFileFormat.HiveMzb)] + public void AFileIsToldByItsExtension(string name, RawFileFormat expected) { + var missingFolder = Path.Combine(_folder, "not-there"); + + Assert.AreEqual(expected, RawFileFormatClassifier.Classify(Path.Combine(missingFolder, name))); + Assert.IsFalse(Directory.Exists(missingFolder), "classifying must not create anything"); + } + + [DataTestMethod()] + [DataRow("sample.iabf")] + [DataRow("sample.ibd")] + [DataRow("sample.mzXML")] + [DataRow("sample.mzData")] + [DataRow("sample.txt")] + [DataRow("sample.raw.zip")] + [DataRow("sample")] + public void AnyOtherExtensionIsUnknown(string name) { + File.WriteAllBytes(Path.Combine(_folder, name), new byte[0]); + + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(Path.Combine(_folder, name))); + } + + [TestMethod()] + public void NoPathIsUnknown() { + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(null)); + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(string.Empty)); + } + + /// + /// A Waters .raw is a folder and a Thermo .raw a file; a .raw that is not there is neither. + /// + [DataTestMethod()] + [DataRow("sample.raw", "file", RawFileFormat.ThermoRaw)] + [DataRow("sample.RAW", "file", RawFileFormat.ThermoRaw)] + [DataRow("sample.raw", "folder", RawFileFormat.WatersRaw)] + [DataRow("sample.RAW", "folder", RawFileFormat.WatersRaw)] + [DataRow("sample.raw", "missing", RawFileFormat.Unknown)] + public void ADotRawIsToldByWhatItIs(string name, string kind, RawFileFormat expected) { + var path = Path.Combine(_folder, name); + if (kind == "file") { + File.WriteAllBytes(path, new byte[0]); + } + else if (kind == "folder") { + Directory.CreateDirectory(path); + } + + Assert.AreEqual(expected, RawFileFormatClassifier.Classify(path)); + } + + /// + /// Agilent and Bruker both write a .d folder. It is checked in the order RawDataHandler reads it: AcqData first, + /// then analysis.tdf, analysis.tsf and analysis.baf. + /// + [DataTestMethod()] + // content: entries separated by "|", a folder name ending in "/"; "" for an empty .d folder, null for no folder at all. + [DataRow("AcqData/", RawFileFormat.AgilentMassHunterD)] + [DataRow("analysis.tdf", RawFileFormat.BrukerTdf)] + [DataRow("analysis.tsf", RawFileFormat.BrukerTsf)] + [DataRow("analysis.baf", RawFileFormat.BrukerBaf)] + [DataRow("AcqData/|analysis.tdf", RawFileFormat.AgilentMassHunterD)] + [DataRow("AcqData/|analysis.baf", RawFileFormat.AgilentMassHunterD)] + [DataRow("analysis.tdf|analysis.tsf|analysis.baf", RawFileFormat.BrukerTdf)] + [DataRow("analysis.tsf|analysis.baf", RawFileFormat.BrukerTsf)] + [DataRow("AcqData", RawFileFormat.Unknown)] + [DataRow("analysis.tdf/", RawFileFormat.Unknown)] + [DataRow("", RawFileFormat.Unknown)] + [DataRow(null, RawFileFormat.Unknown)] + public void ADotDIsToldByWhatItHolds(string content, RawFileFormat expected) { + foreach (var name in new[] { "sample.d", "SAMPLE2.D" }) { + var folder = Path.Combine(_folder, name); + if (content is not null) { + Directory.CreateDirectory(folder); + foreach (var entry in content.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries)) { + if (entry.EndsWith("/")) { + Directory.CreateDirectory(Path.Combine(folder, entry.TrimEnd('/'))); + } + else { + File.WriteAllBytes(Path.Combine(folder, entry), new byte[0]); + } + } + } + + Assert.AreEqual(expected, RawFileFormatClassifier.Classify(folder), name); + } + } + + [DataTestMethod()] + [DataRow("a\0b.raw")] + [DataRow("a\0b.d")] + [DataRow("ac|d.d")] + [DataRow("a\"b.raw")] + public void APathThatCannotBeCheckedIsUnknown(string name) { + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(name)); + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(Path.Combine(_folder, "x") + name)); + } + + [TestMethod()] + public void AVeryLongPathIsUnknownRatherThanAnError() { + var path = Path.Combine(_folder, new string('x', 40000) + ".raw"); + + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Classify(path)); + } + + [TestMethod()] + public void RecordIfUnknownRecordsWhatTheRawDataTell() { + var path = Path.Combine(_folder, "sample.raw"); + File.WriteAllBytes(path, new byte[0]); + var file = new AnalysisFileBean { AnalysisFilePath = path, }; + + Assert.IsTrue(RawFileFormatClassifier.RecordIfUnknown(file)); + Assert.AreEqual(RawFileFormat.ThermoRaw, file.RawFileFormat); + } + + [TestMethod()] + public void RecordIfUnknownNeverReplacesARecordedFormat() { + var path = Path.Combine(_folder, "sample.raw"); + File.WriteAllBytes(path, new byte[0]); + var file = new AnalysisFileBean { AnalysisFilePath = path, RawFileFormat = RawFileFormat.WatersRaw, }; + + Assert.IsFalse(RawFileFormatClassifier.RecordIfUnknown(file)); + Assert.AreEqual(RawFileFormat.WatersRaw, file.RawFileFormat); + } + + [TestMethod()] + public void RecordIfUnknownLeavesAFileThatCannotBeToldUnknown() { + var file = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.raw"), }; + + Assert.IsFalse(RawFileFormatClassifier.RecordIfUnknown(file)); + Assert.AreEqual(RawFileFormat.Unknown, file.RawFileFormat); + Assert.IsFalse(RawFileFormatClassifier.RecordIfUnknown(null)); + } + + [DataTestMethod()] + [DataRow(RawFileFormat.ThermoRaw, ".raw")] + [DataRow(RawFileFormat.WatersRaw, ".RAW")] + [DataRow(RawFileFormat.AgilentMassHunterD, ".d")] + [DataRow(RawFileFormat.BrukerTdf, ".D")] + [DataRow(RawFileFormat.BrukerTsf, ".d")] + [DataRow(RawFileFormat.BrukerBaf, ".d")] + [DataRow(RawFileFormat.SciexWiff, ".wiff")] + [DataRow(RawFileFormat.SciexWiff2, ".WIFF2")] + [DataRow(RawFileFormat.ShimadzuLcd, ".lcd")] + [DataRow(RawFileFormat.ShimadzuQgd, ".QGD")] + [DataRow(RawFileFormat.MzML, ".mzML")] + [DataRow(RawFileFormat.ImzML, ".imzML")] + [DataRow(RawFileFormat.NetCdf, ".CDF")] + [DataRow(RawFileFormat.Abf, ".abf")] + [DataRow(RawFileFormat.Ibf, ".ibf")] + [DataRow(RawFileFormat.KanomaxLrp, ".lrp")] + [DataRow(RawFileFormat.HiveHmd, ".hmd")] + [DataRow(RawFileFormat.HiveMzb, ".mzb")] + public void AFormatIsConsistentOnlyWithItsOwnExtension(RawFileFormat format, string extension) { + Assert.IsTrue(RawFileFormatClassifier.IsConsistentWith(format, extension)); + Assert.IsTrue(RawFileFormatClassifier.IsConsistentWith(format, extension.ToUpperInvariant())); + Assert.IsTrue(RawFileFormatClassifier.IsConsistentWith(format, extension.ToLowerInvariant())); + foreach (var other in new[] { ".raw", ".d", ".wiff", ".wiff2", ".mzml", ".abf", ".txt", "", null }) { + if (!string.Equals(other, extension, StringComparison.OrdinalIgnoreCase)) { + Assert.IsFalse(RawFileFormatClassifier.IsConsistentWith(format, other), $"{format} with {other ?? "null"}"); + } + } + } + + [TestMethod()] + public void UnknownIsConsistentWithNothing() { + foreach (var extension in new[] { ".raw", ".d", ".mzml", ".txt", "", null }) { + Assert.IsFalse(RawFileFormatClassifier.IsConsistentWith(RawFileFormat.Unknown, extension), extension ?? "null"); + } + } + + /// + /// A format added to the enum without an extension would never be trusted by the exporter. + /// + [TestMethod()] + public void EveryKnownFormatHasAnExtension() { + var extensions = new[] { ".raw", ".d", ".wiff", ".wiff2", ".lcd", ".qgd", ".mzml", ".imzml", ".cdf", ".abf", ".ibf", ".lrp", ".hmd", ".mzb" }; + foreach (var format in System.Enum.GetValues(typeof(RawFileFormat)).Cast().Where(f => f != RawFileFormat.Unknown)) { + Assert.IsTrue(extensions.Any(extension => RawFileFormatClassifier.IsConsistentWith(format, extension)), format.ToString()); + } + } + + [TestMethod()] + public void ARecordedFormatDescribesAFileWhoseRawDataAreGone() { + var file = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.d"), RawFileFormat = RawFileFormat.BrukerTdf, }; + + Assert.AreEqual(RawFileFormat.BrukerTdf, RawFileFormatClassifier.Resolve(file)); + } + + [TestMethod()] + public void ARecordedFormatThatDoesNotFitThePathGivesWayToTheDisk() { + var folder = Path.Combine(_folder, "sample.d"); + Directory.CreateDirectory(folder); + File.WriteAllBytes(Path.Combine(folder, "analysis.tdf"), new byte[0]); + var present = new AnalysisFileBean { AnalysisFilePath = folder, RawFileFormat = RawFileFormat.ThermoRaw, }; + var gone = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.d"), RawFileFormat = RawFileFormat.ThermoRaw, }; + + Assert.AreEqual(RawFileFormat.BrukerTdf, RawFileFormatClassifier.Resolve(present)); + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Resolve(gone)); + } + + /// + /// A recorded format is never replaced, so it can outlive the file it was recorded for: another file of the same + /// extension may since have taken its place. A file on disk is described by what is there. + /// + [DataTestMethod()] + // content: as in ADotDIsToldByWhatItHolds, at the path itself; "file" for a plain file there. + [DataRow("sample.raw", "", RawFileFormat.ThermoRaw, RawFileFormat.WatersRaw)] + [DataRow("sample.RAW", "file", RawFileFormat.WatersRaw, RawFileFormat.ThermoRaw)] + [DataRow("sample.d", "analysis.tdf", RawFileFormat.BrukerBaf, RawFileFormat.BrukerTdf)] + [DataRow("sample.d", "AcqData/", RawFileFormat.BrukerTdf, RawFileFormat.AgilentMassHunterD)] + [DataRow("SAMPLE.D", "analysis.baf", RawFileFormat.AgilentMassHunterD, RawFileFormat.BrukerBaf)] + public void AFileOnDiskIsDescribedByWhatItIsWhateverWasRecorded(string name, string content, RawFileFormat recorded, RawFileFormat expected) { + var path = Path.Combine(_folder, name); + if (content == "file") { + File.WriteAllBytes(path, new byte[0]); + } + else { + Directory.CreateDirectory(path); + if (content.EndsWith("/")) { + Directory.CreateDirectory(Path.Combine(path, content.TrimEnd('/'))); + } + else if (content.Length > 0) { + File.WriteAllBytes(Path.Combine(path, content), new byte[0]); + } + } + var file = new AnalysisFileBean { AnalysisFilePath = path, RawFileFormat = recorded, }; + + Assert.AreEqual(expected, RawFileFormatClassifier.Resolve(file)); + Assert.AreEqual(recorded, file.RawFileFormat, "resolving does not change what is recorded"); + } + + /// + /// A vendor folder whose marker is gone but which is still there tells nothing, so the recorded format describes it. + /// + [TestMethod()] + public void AFolderThatNoLongerTellsItsFormatIsDescribedByTheRecord() { + var folder = Path.Combine(_folder, "emptied.d"); + Directory.CreateDirectory(folder); + + Assert.AreEqual(RawFileFormat.BrukerTsf, RawFileFormatClassifier.Resolve(new AnalysisFileBean { AnalysisFilePath = folder, RawFileFormat = RawFileFormat.BrukerTsf, })); + } + + [TestMethod()] + public void NothingRecordedIsToldFromTheDisk() { + var path = Path.Combine(_folder, "sample.raw"); + Directory.CreateDirectory(path); + + Assert.AreEqual(RawFileFormat.WatersRaw, RawFileFormatClassifier.Resolve(new AnalysisFileBean { AnalysisFilePath = path, })); + Assert.AreEqual(RawFileFormat.Unknown, RawFileFormatClassifier.Resolve(new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.raw"), })); + } +} diff --git a/tests/MSDIAL5/MsdialGuiAppTests/Model/DataObj/AnalysisFileBeanModelRawFileFormatTests.cs b/tests/MSDIAL5/MsdialGuiAppTests/Model/DataObj/AnalysisFileBeanModelRawFileFormatTests.cs new file mode 100644 index 000000000..8b8f5f734 --- /dev/null +++ b/tests/MSDIAL5/MsdialGuiAppTests/Model/DataObj/AnalysisFileBeanModelRawFileFormatTests.cs @@ -0,0 +1,123 @@ +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Enum; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; + +namespace CompMs.App.Msdial.Model.DataObj.Tests +{ + /// + /// A project saved before raw data formats were recorded learns each file's format the first time the GUI finds its + /// raw data, and a file pointed at other raw data is described by those. + /// + [TestClass()] + public class AnalysisFileBeanModelRawFileFormatTests + { + private string _folder = string.Empty; + + [TestInitialize()] + public void Setup() { + _folder = Path.Combine(Path.GetTempPath(), $"msdial-analysisfile-format-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_folder); + } + + [TestCleanup()] + public void Cleanup() { + Directory.Delete(_folder, recursive: true); + } + + [TestMethod()] + public void RefreshingFindsAndRecordsTheFormatOfAnOldProjectsFile() { + var bean = new AnalysisFileBean { AnalysisFilePath = CreateThermoRaw("sample.raw"), }; + var model = new AnalysisFileBeanModel(bean); + + Assert.IsTrue(model.RefreshRawDataAvailability()); + + Assert.AreEqual(RawFileFormat.ThermoRaw, bean.RawFileFormat); + } + + [TestMethod()] + public void TheFirstCheckOfTheRawDataAlsoRecordsTheFormat() { + var folder = Path.Combine(_folder, "sample.d"); + Directory.CreateDirectory(folder); + File.WriteAllBytes(Path.Combine(folder, "analysis.tdf"), new byte[0]); + var bean = new AnalysisFileBean { AnalysisFilePath = folder, }; + var model = new AnalysisFileBeanModel(bean); + + Assert.IsTrue(model.IsRawDataAvailable); + + Assert.AreEqual(RawFileFormat.BrukerTdf, bean.RawFileFormat); + } + + [TestMethod()] + public void AFileWhoseRawDataAreMissingStaysUnknown() { + var bean = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone.raw"), }; + var model = new AnalysisFileBeanModel(bean); + + Assert.IsFalse(model.RefreshRawDataAvailability()); + Assert.IsFalse(model.IsRawDataAvailable); + + Assert.AreEqual(RawFileFormat.Unknown, bean.RawFileFormat); + } + + [TestMethod()] + public void ARecordedFormatIsNotReplacedByARefresh() { + var bean = new AnalysisFileBean { AnalysisFilePath = CreateThermoRaw("sample.raw"), RawFileFormat = RawFileFormat.WatersRaw, }; + var model = new AnalysisFileBeanModel(bean); + + Assert.IsTrue(model.RefreshRawDataAvailability()); + + Assert.AreEqual(RawFileFormat.WatersRaw, bean.RawFileFormat); + } + + [TestMethod()] + public void PointingAtOtherRawDataDescribesThem() { + var bean = new AnalysisFileBean { AnalysisFilePath = CreateThermoRaw("thermo.raw"), RawFileFormat = RawFileFormat.ThermoRaw, }; + var model = new AnalysisFileBeanModel(bean); + var waters = Path.Combine(_folder, "waters.raw"); + Directory.CreateDirectory(waters); + + model.AnalysisFilePath = waters; + Assert.AreEqual(RawFileFormat.WatersRaw, bean.RawFileFormat); + + model.AnalysisFilePath = Path.Combine(_folder, "sample.mzML"); + Assert.AreEqual(RawFileFormat.MzML, bean.RawFileFormat); + + // What was recorded for the previous file does not describe one that cannot be told. + model.AnalysisFilePath = Path.Combine(_folder, "gone.d"); + Assert.AreEqual(RawFileFormat.Unknown, bean.RawFileFormat); + } + + [TestMethod()] + public void SettingTheSamePathKeepsTheRecordedFormat() { + var path = Path.Combine(_folder, "gone.raw"); + var bean = new AnalysisFileBean { AnalysisFilePath = path, RawFileFormat = RawFileFormat.ThermoRaw, }; + var model = new AnalysisFileBeanModel(bean); + + model.AnalysisFilePath = path; + + Assert.AreEqual(RawFileFormat.ThermoRaw, bean.RawFileFormat); + } + + [TestMethod()] + public void MarkingTheRawDataMissingKeepsTheRecordedFormat() { + var bean = new AnalysisFileBean { AnalysisFilePath = Path.Combine(_folder, "gone", "sample.d"), RawFileFormat = RawFileFormat.BrukerTdf, }; + var model = new AnalysisFileBeanModel(bean); + + model.MarkRawDataMissing(); + Assert.AreEqual(RawFileFormat.BrukerTdf, bean.RawFileFormat); + + // The collection marks the samples of a folder that is gone without checking each of them. + var collection = new AnalysisFileBeanModelCollection(new[] { model }); + CollectionAssert.AreEqual(new[] { model }, collection.RefreshRawDataAvailability().ToArray()); + Assert.AreEqual(RawFileFormat.BrukerTdf, bean.RawFileFormat); + } + + private string CreateThermoRaw(string name) { + var path = Path.Combine(_folder, name); + File.WriteAllBytes(path, new byte[0]); + return path; + } + } +} diff --git a/tests/MSDIAL5/MsdialGuiAppTests/Model/Setting/DatasetFileSettingModelTests.cs b/tests/MSDIAL5/MsdialGuiAppTests/Model/Setting/DatasetFileSettingModelTests.cs new file mode 100644 index 000000000..ac2520868 --- /dev/null +++ b/tests/MSDIAL5/MsdialGuiAppTests/Model/Setting/DatasetFileSettingModelTests.cs @@ -0,0 +1,51 @@ +using CompMs.MsdialCore.Enum; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; + +namespace CompMs.App.Msdial.Model.Setting.Tests +{ + [TestClass()] + public class DatasetFileSettingModelTests + { + private string _folder = string.Empty; + + [TestInitialize()] + public void Setup() { + _folder = Path.Combine(Path.GetTempPath(), $"msdial-datasetfilesetting-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_folder); + } + + [TestCleanup()] + public void Cleanup() { + Directory.Delete(_folder, recursive: true); + } + + /// + /// The files are registered while they are present, so their formats are recorded then, for mzTab-M to name + /// after the raw data are gone. + /// + [TestMethod()] + public void SetFilesRecordsTheFormatOfEachFile() { + var thermo = Path.Combine(_folder, "a_thermo.raw"); + File.WriteAllBytes(thermo, new byte[0]); + var waters = Path.Combine(_folder, "b_waters.raw"); + Directory.CreateDirectory(waters); + var agilent = Path.Combine(_folder, "c_agilent.d"); + Directory.CreateDirectory(Path.Combine(agilent, "AcqData")); + var bruker = Path.Combine(_folder, "d_bruker.d"); + Directory.CreateDirectory(bruker); + File.WriteAllBytes(Path.Combine(bruker, "analysis.baf"), new byte[0]); + var mzml = Path.Combine(_folder, "e_sample.mzML"); + File.WriteAllBytes(mzml, new byte[0]); + var model = new DatasetFileSettingModel(new DateTime(2026, 10, 16)); + + model.SetFiles(new[] { mzml, bruker, agilent, waters, thermo, }); + + CollectionAssert.AreEqual( + new[] { RawFileFormat.ThermoRaw, RawFileFormat.WatersRaw, RawFileFormat.AgilentMassHunterD, RawFileFormat.BrukerBaf, RawFileFormat.MzML, }, + model.FileModels.AnalysisFiles.Select(file => file.File.RawFileFormat).ToArray()); + } + } +}