From cd2923b321f6be666160f47f05f4e6b6216573cd Mon Sep 17 00:00:00 2001 From: FoolRunning Date: Tue, 24 Oct 2023 10:54:10 -0400 Subject: [PATCH] #460 Changed PDP to also provide USX data --- .../Projects/ParatextProjectDataProvider.cs | 17 ++++ .../ParatextProjectStorageInterpreter.cs | 99 +++++++++++++++++-- c-sharp/Projects/ProjectStorageInterpreter.cs | 8 +- extensions/src/usfm-data-provider/index.d.ts | 30 +++++- src/main/main.ts | 8 +- 5 files changed, 148 insertions(+), 14 deletions(-) diff --git a/c-sharp/Projects/ParatextProjectDataProvider.cs b/c-sharp/Projects/ParatextProjectDataProvider.cs index a50335e4e2..ac3dc94f8a 100644 --- a/c-sharp/Projects/ParatextProjectDataProvider.cs +++ b/c-sharp/Projects/ParatextProjectDataProvider.cs @@ -20,6 +20,9 @@ ProjectDetails projectDetails Getters.Add("getChapterUSFM", GetChapterUSFM); Setters.Add("setChapterUSFM", SetChapterUSFM); Getters.Add("getVerseUSFM", GetVerseUSFM); + + Getters.Add("getChapterUSX", GetChapterUSX); + Setters.Add("setChapterUSX", SetChapterUSX); } protected override Task StartDataProvider() @@ -61,6 +64,7 @@ private ResponseToRequest Set(string dataType, string dataQualifier, string data return _paratextPsi.SetProjectData(scope, data); } + #region USFM handling methods private ResponseToRequest GetBookUSFM(string jsonString) { return Get(ParatextProjectStorageInterpreter.BookUSFM, jsonString); @@ -80,4 +84,17 @@ private ResponseToRequest SetChapterUSFM(string dataQualifier, string data) { return Set(ParatextProjectStorageInterpreter.ChapterUSFM, dataQualifier, data); } + #endregion + + #region USX handling methods + private ResponseToRequest GetChapterUSX(string jsonString) + { + return Get(ParatextProjectStorageInterpreter.ChapterUSX, jsonString); + } + + private ResponseToRequest SetChapterUSX(string dataQualifier, string data) + { + return Set(ParatextProjectStorageInterpreter.ChapterUSX, dataQualifier, data); + } + #endregion } diff --git a/c-sharp/Projects/ParatextProjectStorageInterpreter.cs b/c-sharp/Projects/ParatextProjectStorageInterpreter.cs index 07a7204e06..f7eb8c2f85 100644 --- a/c-sharp/Projects/ParatextProjectStorageInterpreter.cs +++ b/c-sharp/Projects/ParatextProjectStorageInterpreter.cs @@ -1,9 +1,18 @@ +using System.Net.Http.Json; using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Xml; +using System.Xml.Linq; +using System.Xml.XPath; using Newtonsoft.Json; using Paranext.DataProvider.JsonUtils; using Paranext.DataProvider.MessageHandlers; using Paranext.DataProvider.MessageTransports; using Paratext.Data; +using PtxUtils; +using SIL.Scripture; namespace Paranext.DataProvider.Projects; @@ -12,6 +21,7 @@ internal class ParatextProjectStorageInterpreter : ProjectStorageInterpreter public const string BookUSFM = "BookUSFM"; public const string ChapterUSFM = "ChapterUSFM"; public const string VerseUSFM = "VerseUSFM"; + public const string ChapterUSX = "ChapterUSX"; public ParatextProjectStorageInterpreter(PapiClient papiClient) : base(ProjectStorageType.ParatextFolders, new[] { ProjectType.Paratext }, papiClient) { } @@ -88,21 +98,25 @@ public override ResponseToRequest GetProjectData(ProjectDataScope scope) { BookUSFM => string.IsNullOrEmpty(error) - ? ResponseToRequest.Succeeded(scrText.GetText(verseRef, false, false)) + ? ResponseToRequest.Succeeded(scrText.GetText(verseRef, false, true)) : ResponseToRequest.Failed(error), ChapterUSFM => string.IsNullOrEmpty(error) - ? ResponseToRequest.Succeeded(scrText.GetText(verseRef, true, false)) + ? ResponseToRequest.Succeeded(scrText.GetText(verseRef, true, true)) : ResponseToRequest.Failed(error), VerseUSFM => string.IsNullOrEmpty(error) ? ResponseToRequest.Succeeded(scrText.Parser.GetVerseUsfmText(verseRef)) : ResponseToRequest.Failed(error), + ChapterUSX + => string.IsNullOrEmpty(error) + ? ResponseToRequest.Succeeded(GetChapterUsx(scrText, verseRef)) + : ResponseToRequest.Failed(error), _ => ResponseToRequest.Failed($"Unknown data type: {scope.DataType}") }; } - public override ResponseToRequest SetProjectData(ProjectDataScope scope, string data) + public override ResponseToRequest SetProjectData(ProjectDataScope scope, JsonNode data) { if (scope.ProjectID == null) return ResponseToRequest.Failed("Must provide a project ID"); @@ -128,13 +142,23 @@ public override ResponseToRequest SetProjectData(ProjectDataScope scope, string verseRef.BookNum, verseRef.ChapterNum, false, - data, + data.ToString(), writeLock ); } ); // The value of returned string is case sensitive and cannot change unless data provider subscriptions change return ResponseToRequest.Succeeded(ChapterUSFM); + case ChapterUSX: + if (!string.IsNullOrEmpty(error)) + return ResponseToRequest.Failed(error); + ResponseToRequest? response = null; + RunWithinLock( + WriteScope.ProjectText(scrText, verseRef.BookNum, verseRef.ChapterNum), + writeLock => + response = SetChapterUsx(scrText, verseRef, data.ToString(), writeLock) + ); + return response ?? ResponseToRequest.Failed("Unknown error occurred"); default: return ResponseToRequest.Failed($"Unknown data type: {scope.DataType}"); } @@ -157,7 +181,7 @@ public override ResponseToRequest GetExtensionData(ProjectDataScope scope) return ResponseToRequest.Succeeded(textReader.ReadToEnd()); } - public override ResponseToRequest SetExtensionData(ProjectDataScope scope, string data) + public override ResponseToRequest SetExtensionData(ProjectDataScope scope, JsonNode data) { if (scope.ProjectID == null) return ResponseToRequest.Failed("Must provide a project ID"); @@ -181,7 +205,7 @@ public override ResponseToRequest SetExtensionData(ProjectDataScope scope, strin throw new Exception("Write lock is not active"); dataStream.SetLength(0); using TextWriter textWriter = new StreamWriter(dataStream, Encoding.UTF8); - textWriter.Write(data); + textWriter.Write(data.ToString()); textWriter.Flush(); } ); @@ -204,6 +228,69 @@ public override ResponseToRequest SetExtensionData(ProjectDataScope scope, strin ); } + private string GetChapterUsx(ScrText scrText, VerseRef vref) + { + XmlDocument usx = ConvertUsfmToUsx( + scrText, + scrText.GetText(vref, true, true), + vref.BookNum + ); + return usx.OuterXml ?? string.Empty; + } + + private ResponseToRequest SetChapterUsx( + ScrText scrText, + VerseRef vref, + string newUsx, + WriteLock writeLock + ) + { + try + { + XDocument doc; + using (TextReader reader = new StringReader(newUsx)) + doc = XDocument.Load(reader, LoadOptions.PreserveWhitespace); + + if (doc.Root?.Name != "usx") + return ResponseToRequest.Failed("Invalid USX"); + + UsxFragmenter.FindFragments( + scrText.ScrStylesheet(vref.BookNum), + doc.CreateNavigator(), + XPathExpression.Compile("*[false()]"), + out string usfm + ); + + usfm = UsfmToken.NormalizeUsfm(scrText, vref.BookNum, usfm); + scrText.PutText(vref.BookNum, vref.ChapterNum, true, usfm, writeLock); + } + catch (Exception e) + { + return ResponseToRequest.Failed(e.ToString()); + } + + return ResponseToRequest.Succeeded(ChapterUSX); + } + + /// + /// Converts usfm to usx, but does not annotate + /// + private XmlDocument ConvertUsfmToUsx(ScrText scrText, string usfm, int bookNum) + { + ScrStylesheet scrStylesheet = scrText.ScrStylesheet(bookNum); + // Tokenize usfm + List tokens = UsfmToken.Tokenize(scrStylesheet, usfm ?? string.Empty, true); + + XmlDocument doc = new XmlDocument(); + using (XmlWriter xmlw = doc.CreateNavigator()!.AppendChild()) + { + // Convert to XML + UsfmToUsx.ConvertToXmlWriter(scrStylesheet, tokens, xmlw, false); + xmlw.Flush(); + } + return doc; + } + private static void RunWithinLock(WriteScope writeScope, Action action) { var myLock = diff --git a/c-sharp/Projects/ProjectStorageInterpreter.cs b/c-sharp/Projects/ProjectStorageInterpreter.cs index 8c7d109e43..f488053dc9 100644 --- a/c-sharp/Projects/ProjectStorageInterpreter.cs +++ b/c-sharp/Projects/ProjectStorageInterpreter.cs @@ -55,13 +55,13 @@ out string errorMessage case "getProjectData": return GetProjectData(dataScope); case "setProjectData": - var setProjectReturn = SetProjectData(dataScope, args[1]!.ToJsonString()); + var setProjectReturn = SetProjectData(dataScope, args[1]!); SendDataUpdateEvent(setProjectReturn.Contents); return setProjectReturn; case "getExtensionData": return GetExtensionData(dataScope); case "setExtensionData": - var setExtensionReturn = SetExtensionData(dataScope, args[1]!.ToJsonString()); + var setExtensionReturn = SetExtensionData(dataScope, args[1]!); SendDataUpdateEvent(setExtensionReturn.Contents); return setExtensionReturn; default: @@ -106,7 +106,7 @@ private ResponseToRequest GetSupportedTypes() /// /// Set data in a project identified by . /// - public abstract ResponseToRequest SetProjectData(ProjectDataScope scope, string jsonData); + public abstract ResponseToRequest SetProjectData(ProjectDataScope scope, JsonNode data); /// /// Get an extension's data in a project identified by . @@ -116,5 +116,5 @@ private ResponseToRequest GetSupportedTypes() /// /// Set an extension's data in a project identified by . /// - public abstract ResponseToRequest SetExtensionData(ProjectDataScope scope, string jsonData); + public abstract ResponseToRequest SetExtensionData(ProjectDataScope scope, JsonNode data); } diff --git a/extensions/src/usfm-data-provider/index.d.ts b/extensions/src/usfm-data-provider/index.d.ts index 6c8f0a85b0..d4b24ec5bb 100644 --- a/extensions/src/usfm-data-provider/index.d.ts +++ b/extensions/src/usfm-data-provider/index.d.ts @@ -29,12 +29,14 @@ declare module 'papi-shared-types' { * This is not yet a complete list of the data types available from Paratext projects. */ export type ParatextStandardProjectDataTypes = MandatoryProjectDataType & { - /** Gets the "raw" USFM data for the specified book */ + /** Gets/sets the "raw" USFM data for the specified book */ BookUSFM: DataProviderDataType; - /** Gets the "raw" USFM data for the specified chapter */ + /** Gets/sets the "raw" USFM data for the specified chapter */ ChapterUSFM: DataProviderDataType; - /** Gets the "raw" USFM data for the specified verse */ + /** Gets/sets the "raw" USFM data for the specified verse */ VerseUSFM: DataProviderDataType; + /** Gets/sets the data in USX form for the specified chapter */ + ChapterUSX: DataProviderDataType; /** * Gets the tokenized USJ data for the specified book * @@ -188,6 +190,28 @@ declare module 'papi-shared-types' { options?: DataProviderSubscriberOptions, ): Unsubscriber; + /** Gets the Scripture text in USX format for the specified chapter */ + getChapterUSX(verseRef: VerseRef): Promise; + /** Sets the Scripture text in USX format for the specified chapter */ + setChapterUSX( + verseRef: VerseRef, + usx: string, + ): Promise>; + /** + * Subscribe to run a callback function when the USX data is changed + * + * @param verseRef tells the provider what changes to listen for + * @param callback function to run with the updated USX for this selector + * @param options various options to adjust how the subscriber emits updates + * + * @returns unsubscriber function (run to unsubscribe from listening for updates) + */ + subscribeVerseUSX( + verseRef: VerseRef, + callback: (usx: string | undefined) => void, + options?: DataProviderSubscriberOptions, + ): Unsubscriber; + /** * Gets the tokenized USJ data for the specified book * diff --git a/src/main/main.ts b/src/main/main.ts index 41bf013df1..09eb33c3d0 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -23,6 +23,9 @@ import extensionAssetProtocolService from '@main/services/extension-asset-protoc import { wait } from '@shared/utils/util'; import { CommandNames } from 'papi-shared-types'; import { SerializedRequestType } from '@shared/utils/papi-util'; +// Used with the commented out code at the bottom of this file to test the ParatextProjectDataProvider +// import { getProjectDataProvider } from '@shared/services/project-data-provider.service'; +// import { VerseRef } from '@sillsdev/scripture'; const PROCESS_CLOSE_TIME_OUT = 2000; @@ -324,8 +327,11 @@ async function main() { const paratextPdp = await getProjectDataProvider<'ParatextStandard'>( '32664dc3288a28df2e2bb75ded887fc8f17a15fb', ); - const verse = await paratextPdp.getVerseUSFM(new VerseRef('JHN', '1', '1')); + const verse = await paratextPdp.getChapterUSX(new VerseRef('JHN', '1', '1')); logger.info(`Got PDP data: ${verse}`); + + if (verse !== undefined) await paratextPdp.setChapterUSX(new VerseRef('JHN', '1', '1'), verse); + paratextPdp.setExtensionData( { extensionName: 'foo', dataQualifier: 'fooData' }, 'This is the data from extension foo',