From 5998819f8fb6f1b695132f1c44e8c06891d17029 Mon Sep 17 00:00:00 2001 From: Stamina9 <1934687883@qq.com> Date: Wed, 28 Oct 2026 16:32:44 +0800 Subject: [PATCH] feat(data): convert FreeMASTER oscilloscope text to CSV Recognize the oscilloscope marker and tab-separated measurements, preserve their text as CSV, and offer the existing CSV preset on text donuts. Addresses #571. --- docs/ENGINES.md | 2 +- src/Filee.Core/Presets/BuiltInData.cs | 2 +- src/Filee.Core/Settings/AppSettings.cs | 2 +- src/Filee.Core/Settings/UserDataStore.cs | 8 +++ src/Filee.Engines/Data/DataConverter.cs | 12 +++++ src/Filee.Engines/Data/FreeMasterText.cs | 24 +++++++++ .../FreeMasterProfileTests.cs | 40 +++++++++++++++ .../FreeMasterTextTests.cs | 49 +++++++++++++++++++ 8 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 src/Filee.Engines/Data/FreeMasterText.cs create mode 100644 tests/Filee.Core.Tests/FreeMasterProfileTests.cs create mode 100644 tests/Filee.Engines.Tests/FreeMasterTextTests.cs diff --git a/docs/ENGINES.md b/docs/ENGINES.md index 71ac50c3f..a00b4332f 100644 --- a/docs/ENGINES.md +++ b/docs/ENGINES.md @@ -22,7 +22,7 @@ Filee chooses engines automatically. Settings → *Engines* shows their status, | PDF slides | built in (PDFium, PdfPig) | **PDF → PPTX**: a picture of each page on its own slide (looks like the PDF, not editable), the page's text in the speaker notes | | PDF tables | built in (PdfPig) | **PDF → XLSX, ODS, CSV** (#267): each table drawn with lines on its own sheet, numbers and dates as real values, merged cells and column widths kept | | E-mail | built in (MimeKit, MsgReader) | **EML, Outlook MSG → HTML** (headers + body, inline pictures), **TXT**, **ZIP** (the attachments); **MSG → EML**; **MHTML → HTML** (the saved page in one file, pictures inlined; on to PDF, DOCX, HWPX, TXT, Markdown, EPUB from there) | -| Data | built in (System.Text.Json, YamlDotNet, Tomlyn, System.Xml) | **JSON, JSON Lines, YAML, TOML, XML, CSV ↔ each other** (CSV for tables); **vCard, iCalendar ↔ CSV**; spreadsheets, contacts and events reach JSON / YAML / TOML / XML through CSV | +| Data | built in (System.Text.Json, YamlDotNet, Tomlyn, System.Xml) | **JSON, JSON Lines, YAML, TOML, XML, CSV ↔ each other** (CSV for tables); **vCard, iCalendar ↔ CSV**; FreeMASTER oscilloscope TXT → CSV; spreadsheets, contacts and events reach JSON / YAML / TOML / XML through CSV | | **E-books** | built in | **EPUB, MOBI/AZW/AZW3/PRC, FB2, HTMLZ, TXTZ → EPUB, HWPX (→ PDF), TXT, HTML, Markdown, FB2, HTMLZ, TXTZ**; HTML, Markdown, TXT, DOCX → EPUB/FB2/HTMLZ/TXTZ; HTML → TXT; comics CBZ/CBR/CB7/CBT/CBC → PDF, EPUB, CBZ; PDF → CBZ; AZW4 → PDF; **EPUB ↔ KEPUB** (Kobo) | | CAD (ACadSharp + built-in renderer) | built in (library) | **DWG ↔ DXF**; DWG/DXF → **PDF** (vector), **SVG**, PNG, JPG, WEBP, TIFF, BMP, GIF, ICO, AVIF | | **Subtitles** | built in | **SRT, ASS/SSA, WebVTT, SAMI (SMI), LRC, TTML, MicroDVD, SCC → SRT, ASS, WebVTT, SAMI, LRC, TTML, MicroDVD**, always UTF-8 (Korean CP949 / EUC-KR files read correctly) | diff --git a/src/Filee.Core/Presets/BuiltInData.cs b/src/Filee.Core/Presets/BuiltInData.cs index ffab9bc78..ff5ae591c 100644 --- a/src/Filee.Core/Presets/BuiltInData.cs +++ b/src/Filee.Core/Presets/BuiltInData.cs @@ -651,7 +651,7 @@ .. WithSubtitleProfile(CatalogProfiles()), NameKey = "profile.text", Extensions = ExtensionsOf(FormatCategory.Text), PresetIds = ["to-pdf", "to-hwpx", "to-hwp", "to-docx", "to-html", "to-txt", "to-epub"], - BackPresetIds = ["to-md", "to-odt", "to-rtf", "to-kepub", "merge-pdf", "zip-all"], + BackPresetIds = ["to-md", "to-odt", "to-rtf", "to-kepub", "to-csv", "merge-pdf", "zip-all"], }; /// Donuts of the categories added in settings schema 6, in the order they follow "text". diff --git a/src/Filee.Core/Settings/AppSettings.cs b/src/Filee.Core/Settings/AppSettings.cs index f32156ead..08f12f759 100644 --- a/src/Filee.Core/Settings/AppSettings.cs +++ b/src/Filee.Core/Settings/AppSettings.cs @@ -123,7 +123,7 @@ public static bool IsFlipKeyAllowed(string key, IEnumerable gest public sealed class AppSettings { /// Current schema version. Increase when the format changes and add a migration. - public const int CurrentSchemaVersion = 54; + public const int CurrentSchemaVersion = 55; public int SchemaVersion { get; set; } = CurrentSchemaVersion; diff --git a/src/Filee.Core/Settings/UserDataStore.cs b/src/Filee.Core/Settings/UserDataStore.cs index b788b6903..350d72e13 100644 --- a/src/Filee.Core/Settings/UserDataStore.cs +++ b/src/Filee.Core/Settings/UserDataStore.cs @@ -931,6 +931,14 @@ public static bool ApplyToLibrary(int from, List presets, List p.Id == "to-csv") + && profiles.FirstOrDefault(p => p.Id == "text") is { BackPresetIds: { } textBack } textProfile + && !textProfile.PresetIds.Contains("to-csv") && !textBack.Contains("to-csv") && textBack.Count < ToolbarProfile.MaxSlices) + { + InsertBefore(textBack, "to-csv", "merge-pdf"); + changed = true; + } return changed; } diff --git a/src/Filee.Engines/Data/DataConverter.cs b/src/Filee.Engines/Data/DataConverter.cs index 5ded238d4..d115248fd 100644 --- a/src/Filee.Engines/Data/DataConverter.cs +++ b/src/Filee.Engines/Data/DataConverter.cs @@ -32,6 +32,7 @@ public sealed class DataConverter : IConverter .. Trees.Append("csv").Select(to => new ConversionEdge("har", to)), new("ics", "csv"), new("csv", "ics"), + new("txt", "csv", DirectOnly: true), ]; public EngineStatus GetStatus() => @@ -45,6 +46,17 @@ public Task> ConvertAsync(ConversionStep step, IProgress>(() => { progress?.Report(0.1); + if (step.From == "txt" && step.To == "csv") + { + var source = HwpxConverter.DecodeText(File.ReadAllBytes(step.InputPath)); + var table = FreeMasterText.Read(source); + cancellationToken.ThrowIfCancellationRequested(); + if (step.Output.Allocate("csv") is not { } csv) + return []; + CsvFormat.WriteRows(table, csv); + progress?.Report(1); + return [csv]; + } if (step.From == "vcf") { var contacts = VCard.Read(File.ReadAllBytes(step.InputPath)); diff --git a/src/Filee.Engines/Data/FreeMasterText.cs b/src/Filee.Engines/Data/FreeMasterText.cs new file mode 100644 index 000000000..0297f5eaf --- /dev/null +++ b/src/Filee.Engines/Data/FreeMasterText.cs @@ -0,0 +1,24 @@ +// FreeMASTER oscilloscope TXT exports: a marker, a tab-separated header and tab-separated measurement rows. + +using Filee.Engines.Office.Sheets; + +namespace Filee.Engines.Data; + +internal static class FreeMasterText +{ + public static List> Read(string text) + { + var rows = CsvFormat.Parse(text.TrimStart('\uFEFF'), CsvFormat.Tab) + .Where(row => row.Count != 1 || row[0].Length != 0).ToList(); + if (rows.Count < 3 || rows[0].Count != 1 || rows[0][0].Trim() != "# Oscilloscope Data" + || rows[1].Count < 2 || rows[1][0].Trim() != "# Seconds") + throw new InvalidDataException("Expected a FreeMASTER oscilloscope export with a tab-separated header and data."); + + var header = rows[1]; + header[0] = "Seconds"; + var result = rows.Skip(1).ToList(); + if (result.Skip(1).Any(row => row.Count != header.Count)) + throw new InvalidDataException("A FreeMASTER data row has a different number of columns from the header."); + return result; + } +} diff --git a/tests/Filee.Core.Tests/FreeMasterProfileTests.cs b/tests/Filee.Core.Tests/FreeMasterProfileTests.cs new file mode 100644 index 000000000..39e430730 --- /dev/null +++ b/tests/Filee.Core.Tests/FreeMasterProfileTests.cs @@ -0,0 +1,40 @@ +// Existing text donuts can offer FreeMASTER TXT → CSV without changing custom layouts. + +using Filee.Core.Presets; +using Filee.Core.Profiles; +using Filee.Core.Settings; + +namespace Filee.Core.Tests; + +public class FreeMasterProfileTests +{ + [Fact] + public void New_text_profile_offers_csv() + { + Assert.Contains("to-csv", BuiltInData.TextProfile().BackPresetIds!); + } + + [Fact] + public void Migration_offers_csv_only_once_and_respects_existing_choices() + { + var presets = BuiltInData.CreatePresets(); + var profiles = BuiltInData.CreateProfiles(); + var text = profiles.Single(p => p.Id == "text"); + text.BackPresetIds!.Remove("to-csv"); + Assert.True(SettingsMigrations.ApplyToLibrary(54, presets, profiles)); + Assert.Contains("to-csv", text.BackPresetIds); + Assert.False(SettingsMigrations.ApplyToLibrary(54, presets, profiles)); + + text.BackPresetIds.Remove("to-csv"); + text.PresetIds.Add("to-csv"); + Assert.False(SettingsMigrations.ApplyToLibrary(54, presets, profiles)); + Assert.DoesNotContain("to-csv", text.BackPresetIds); + + text.PresetIds.Remove("to-csv"); + while (text.BackPresetIds.Count < ToolbarProfile.MaxSlices) + text.BackPresetIds.Add("user-" + text.BackPresetIds.Count); + Assert.False(SettingsMigrations.ApplyToLibrary(54, presets, profiles)); + Assert.DoesNotContain("to-csv", text.BackPresetIds); + Assert.False(SettingsMigrations.ApplyToLibrary(55, presets, profiles)); + } +} diff --git a/tests/Filee.Engines.Tests/FreeMasterTextTests.cs b/tests/Filee.Engines.Tests/FreeMasterTextTests.cs new file mode 100644 index 000000000..07679fbda --- /dev/null +++ b/tests/Filee.Engines.Tests/FreeMasterTextTests.cs @@ -0,0 +1,49 @@ +// FreeMASTER oscilloscope text exports become CSV without interpreting measurement values. + +using System.Text; +using Filee.Core.Conversion; +using Filee.Core.Presets; +using Filee.Engines.Office.Sheets; + +namespace Filee.Engines.Tests; + +public class FreeMasterTextTests(EngineFixture fx) : IClassFixture +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Oscilloscope_export_becomes_csv_with_text_and_empty_fields(bool utf16) + { + var path = Path.Combine(fx.NewFolder(), "scope.txt"); + const string input = "# Oscilloscope Data\r\n\r\n# Seconds\tLabel\tValue\tEmpty\r\n" + + "0.000\t\"cell, \"\"A\"\"\"\t4.12300\t\r\n0.541\t한글\t007\t\r\n"; + await File.WriteAllTextAsync(path, input, utf16 ? Encoding.Unicode : new UTF8Encoding(false), TestContext.Current.CancellationToken); + + var job = await fx.ConvertAsync([path], new Preset { TargetFormat = "csv" }); + + Assert.True(job.State == JobState.Completed, string.Join("; ", job.Files.Select(f => f.ErrorDetail))); + var bytes = await File.ReadAllBytesAsync(Assert.Single(job.Outputs), TestContext.Current.CancellationToken); + Assert.True(bytes is [0xEF, 0xBB, 0xBF, ..]); + var rows = CsvFormat.Parse(Encoding.UTF8.GetString(bytes).TrimStart('\uFEFF'), ','); + Assert.Equal(new[] { "Seconds", "Label", "Value", "Empty" }, rows[0]); + Assert.Equal(new[] { "0.000", "cell, \"A\"", "4.12300", "" }, rows[1]); + Assert.Equal(new[] { "0.541", "한글", "007", "" }, rows[2]); + Assert.Equal(3, rows.Count); + } + + [Theory] + [InlineData("ordinary plain text\nwithout table\n")] + [InlineData("# Oscilloscope Data\n# Seconds\tValue\n0.0\t1\t2\n")] + [InlineData("# Oscilloscope Data\n# Seconds\tValue\n")] + public async Task Non_matching_or_incomplete_data_is_refused(string input) + { + var path = Path.Combine(fx.NewFolder(), "scope.txt"); + await File.WriteAllTextAsync(path, input, TestContext.Current.CancellationToken); + + var job = await fx.ConvertAsync([path], new Preset { TargetFormat = "csv" }); + + Assert.Equal(JobState.Failed, job.State); + Assert.Empty(job.Outputs); + Assert.Equal(input, await File.ReadAllTextAsync(path, TestContext.Current.CancellationToken)); + } +}