Skip to content

Commit

Permalink
#460 Changed PDP to also provide USX data
Browse files Browse the repository at this point in the history
  • Loading branch information
@FoolRunning
FoolRunning committed Oct 24, 2023
1 parent d86f4e2 commit cd2923b
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 14 deletions.
17 changes: 17 additions & 0 deletions c-sharp/Projects/ParatextProjectDataProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ ProjectDetails projectDetails
Getters.Add("getChapterUSFM", GetChapterUSFM);
Setters.Add("setChapterUSFM", SetChapterUSFM);
Getters.Add("getVerseUSFM", GetVerseUSFM);

Getters.Add("getChapterUSX", GetChapterUSX);
Setters.Add("setChapterUSX", SetChapterUSX);
}

protected override Task StartDataProvider()
Expand Down Expand Up @@ -61,6 +64,7 @@ private ResponseToRequest Set(string dataType, string dataQualifier, string data
return _paratextPsi.SetProjectData(scope, data);
}

#region USFM handling methods
private ResponseToRequest GetBookUSFM(string jsonString)
{
return Get(ParatextProjectStorageInterpreter.BookUSFM, jsonString);
Expand All @@ -80,4 +84,17 @@ private ResponseToRequest SetChapterUSFM(string dataQualifier, string data)
{
return Set(ParatextProjectStorageInterpreter.ChapterUSFM, dataQualifier, data);
}
#endregion

#region USX handling methods
private ResponseToRequest GetChapterUSX(string jsonString)
{
return Get(ParatextProjectStorageInterpreter.ChapterUSX, jsonString);
}

private ResponseToRequest SetChapterUSX(string dataQualifier, string data)
{
return Set(ParatextProjectStorageInterpreter.ChapterUSX, dataQualifier, data);
}
#endregion
}
99 changes: 93 additions & 6 deletions c-sharp/Projects/ParatextProjectStorageInterpreter.cs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,18 @@
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Linq;
using System.Xml.XPath;
using Newtonsoft.Json;
using Paranext.DataProvider.JsonUtils;
using Paranext.DataProvider.MessageHandlers;
using Paranext.DataProvider.MessageTransports;
using Paratext.Data;
using PtxUtils;
using SIL.Scripture;

namespace Paranext.DataProvider.Projects;

Expand All @@ -12,6 +21,7 @@ internal class ParatextProjectStorageInterpreter : ProjectStorageInterpreter
public const string BookUSFM = "BookUSFM";
public const string ChapterUSFM = "ChapterUSFM";
public const string VerseUSFM = "VerseUSFM";
public const string ChapterUSX = "ChapterUSX";

public ParatextProjectStorageInterpreter(PapiClient papiClient)
: base(ProjectStorageType.ParatextFolders, new[] { ProjectType.Paratext }, papiClient) { }
Expand Down Expand Up @@ -88,21 +98,25 @@ public override ResponseToRequest GetProjectData(ProjectDataScope scope)
{
BookUSFM
=> string.IsNullOrEmpty(error)
? ResponseToRequest.Succeeded(scrText.GetText(verseRef, false, false))
? ResponseToRequest.Succeeded(scrText.GetText(verseRef, false, true))
: ResponseToRequest.Failed(error),
ChapterUSFM
=> string.IsNullOrEmpty(error)
? ResponseToRequest.Succeeded(scrText.GetText(verseRef, true, false))
? ResponseToRequest.Succeeded(scrText.GetText(verseRef, true, true))
: ResponseToRequest.Failed(error),
VerseUSFM
=> string.IsNullOrEmpty(error)
? ResponseToRequest.Succeeded(scrText.Parser.GetVerseUsfmText(verseRef))
: ResponseToRequest.Failed(error),
ChapterUSX
=> string.IsNullOrEmpty(error)
? ResponseToRequest.Succeeded(GetChapterUsx(scrText, verseRef))
: ResponseToRequest.Failed(error),
_ => ResponseToRequest.Failed($"Unknown data type: {scope.DataType}")
};
}

public override ResponseToRequest SetProjectData(ProjectDataScope scope, string data)
public override ResponseToRequest SetProjectData(ProjectDataScope scope, JsonNode data)
{
if (scope.ProjectID == null)
return ResponseToRequest.Failed("Must provide a project ID");
Expand All @@ -128,13 +142,23 @@ public override ResponseToRequest SetProjectData(ProjectDataScope scope, string
verseRef.BookNum,
verseRef.ChapterNum,
false,
data,
data.ToString(),
writeLock
);
}
);
// The value of returned string is case sensitive and cannot change unless data provider subscriptions change
return ResponseToRequest.Succeeded(ChapterUSFM);
case ChapterUSX:
if (!string.IsNullOrEmpty(error))
return ResponseToRequest.Failed(error);
ResponseToRequest? response = null;
RunWithinLock(
WriteScope.ProjectText(scrText, verseRef.BookNum, verseRef.ChapterNum),
writeLock =>
response = SetChapterUsx(scrText, verseRef, data.ToString(), writeLock)
);
return response ?? ResponseToRequest.Failed("Unknown error occurred");
default:
return ResponseToRequest.Failed($"Unknown data type: {scope.DataType}");
}
Expand All @@ -157,7 +181,7 @@ public override ResponseToRequest GetExtensionData(ProjectDataScope scope)
return ResponseToRequest.Succeeded(textReader.ReadToEnd());
}

public override ResponseToRequest SetExtensionData(ProjectDataScope scope, string data)
public override ResponseToRequest SetExtensionData(ProjectDataScope scope, JsonNode data)
{
if (scope.ProjectID == null)
return ResponseToRequest.Failed("Must provide a project ID");
Expand All @@ -181,7 +205,7 @@ public override ResponseToRequest SetExtensionData(ProjectDataScope scope, strin
throw new Exception("Write lock is not active");
dataStream.SetLength(0);
using TextWriter textWriter = new StreamWriter(dataStream, Encoding.UTF8);
textWriter.Write(data);
textWriter.Write(data.ToString());
textWriter.Flush();
}
);
Expand All @@ -204,6 +228,69 @@ public override ResponseToRequest SetExtensionData(ProjectDataScope scope, strin
);
}

private string GetChapterUsx(ScrText scrText, VerseRef vref)
{
XmlDocument usx = ConvertUsfmToUsx(
scrText,
scrText.GetText(vref, true, true),
vref.BookNum
);
return usx.OuterXml ?? string.Empty;
}

private ResponseToRequest SetChapterUsx(
ScrText scrText,
VerseRef vref,
string newUsx,
WriteLock writeLock
)
{
try
{
XDocument doc;
using (TextReader reader = new StringReader(newUsx))
doc = XDocument.Load(reader, LoadOptions.PreserveWhitespace);

if (doc.Root?.Name != "usx")
return ResponseToRequest.Failed("Invalid USX");

UsxFragmenter.FindFragments(
scrText.ScrStylesheet(vref.BookNum),
doc.CreateNavigator(),
XPathExpression.Compile("*[false()]"),
out string usfm
);

usfm = UsfmToken.NormalizeUsfm(scrText, vref.BookNum, usfm);
scrText.PutText(vref.BookNum, vref.ChapterNum, true, usfm, writeLock);
}
catch (Exception e)
{
return ResponseToRequest.Failed(e.ToString());
}

return ResponseToRequest.Succeeded(ChapterUSX);
}

/// <summary>
/// Converts usfm to usx, but does not annotate
/// </summary>
private XmlDocument ConvertUsfmToUsx(ScrText scrText, string usfm, int bookNum)
{
ScrStylesheet scrStylesheet = scrText.ScrStylesheet(bookNum);
// Tokenize usfm
List<UsfmToken> tokens = UsfmToken.Tokenize(scrStylesheet, usfm ?? string.Empty, true);

XmlDocument doc = new XmlDocument();
using (XmlWriter xmlw = doc.CreateNavigator()!.AppendChild())
{
// Convert to XML
UsfmToUsx.ConvertToXmlWriter(scrStylesheet, tokens, xmlw, false);
xmlw.Flush();
}
return doc;
}

private static void RunWithinLock(WriteScope writeScope, Action<WriteLock> action)
{
var myLock =
Expand Down
8 changes: 4 additions & 4 deletions c-sharp/Projects/ProjectStorageInterpreter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,13 @@ out string errorMessage
case "getProjectData":
return GetProjectData(dataScope);
case "setProjectData":
var setProjectReturn = SetProjectData(dataScope, args[1]!.ToJsonString());
var setProjectReturn = SetProjectData(dataScope, args[1]!);
SendDataUpdateEvent(setProjectReturn.Contents);
return setProjectReturn;
case "getExtensionData":
return GetExtensionData(dataScope);
case "setExtensionData":
var setExtensionReturn = SetExtensionData(dataScope, args[1]!.ToJsonString());
var setExtensionReturn = SetExtensionData(dataScope, args[1]!);
SendDataUpdateEvent(setExtensionReturn.Contents);
return setExtensionReturn;
default:
Expand Down Expand Up @@ -106,7 +106,7 @@ private ResponseToRequest GetSupportedTypes()
/// <summary>
/// Set data in a project identified by <param name="scope"></param>.
/// </summary>
public abstract ResponseToRequest SetProjectData(ProjectDataScope scope, string jsonData);
public abstract ResponseToRequest SetProjectData(ProjectDataScope scope, JsonNode data);

/// <summary>
/// Get an extension's data in a project identified by <param name="scope"></param>.
Expand All @@ -116,5 +116,5 @@ private ResponseToRequest GetSupportedTypes()
/// <summary>
/// Set an extension's data in a project identified by <param name="scope"></param>.
/// </summary>
public abstract ResponseToRequest SetExtensionData(ProjectDataScope scope, string jsonData);
public abstract ResponseToRequest SetExtensionData(ProjectDataScope scope, JsonNode data);
}
30 changes: 27 additions & 3 deletions extensions/src/usfm-data-provider/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,14 @@ declare module 'papi-shared-types' {
* This is not yet a complete list of the data types available from Paratext projects.
*/
export type ParatextStandardProjectDataTypes = MandatoryProjectDataType & {
/** Gets the "raw" USFM data for the specified book */
/** Gets/sets the "raw" USFM data for the specified book */
BookUSFM: DataProviderDataType<VerseRef, string | undefined, string>;
/** Gets the "raw" USFM data for the specified chapter */
/** Gets/sets the "raw" USFM data for the specified chapter */
ChapterUSFM: DataProviderDataType<VerseRef, string | undefined, string>;
/** Gets the "raw" USFM data for the specified verse */
/** Gets/sets the "raw" USFM data for the specified verse */
VerseUSFM: DataProviderDataType<VerseRef, string | undefined, string>;
/** Gets/sets the data in USX form for the specified chapter */
ChapterUSX: DataProviderDataType<VerseRef, string | undefined, string>;
/**
* Gets the tokenized USJ data for the specified book
*
Expand Down Expand Up @@ -188,6 +190,28 @@ declare module 'papi-shared-types' {
options?: DataProviderSubscriberOptions,
): Unsubscriber;

/** Gets the Scripture text in USX format for the specified chapter */
getChapterUSX(verseRef: VerseRef): Promise<string | undefined>;
/** Sets the Scripture text in USX format for the specified chapter */
setChapterUSX(
verseRef: VerseRef,
usx: string,
): Promise<DataProviderUpdateInstructions<ParatextStandardProjectDataTypes>>;
/**
* Subscribe to run a callback function when the USX data is changed
*
* @param verseRef tells the provider what changes to listen for
* @param callback function to run with the updated USX for this selector
* @param options various options to adjust how the subscriber emits updates
*
* @returns unsubscriber function (run to unsubscribe from listening for updates)
*/
subscribeVerseUSX(
verseRef: VerseRef,
callback: (usx: string | undefined) => void,
options?: DataProviderSubscriberOptions,
): Unsubscriber;

/**
* Gets the tokenized USJ data for the specified book
*
Expand Down
8 changes: 7 additions & 1 deletion src/main/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ import extensionAssetProtocolService from '@main/services/extension-asset-protoc
import { wait } from '@shared/utils/util';
import { CommandNames } from 'papi-shared-types';
import { SerializedRequestType } from '@shared/utils/papi-util';
// Used with the commented out code at the bottom of this file to test the ParatextProjectDataProvider
// import { getProjectDataProvider } from '@shared/services/project-data-provider.service';
// import { VerseRef } from '@sillsdev/scripture';

const PROCESS_CLOSE_TIME_OUT = 2000;

Expand Down Expand Up @@ -324,8 +327,11 @@ async function main() {
const paratextPdp = await getProjectDataProvider<'ParatextStandard'>(
'32664dc3288a28df2e2bb75ded887fc8f17a15fb',
);
const verse = await paratextPdp.getVerseUSFM(new VerseRef('JHN', '1', '1'));
const verse = await paratextPdp.getChapterUSX(new VerseRef('JHN', '1', '1'));
logger.info(`Got PDP data: ${verse}`);
if (verse !== undefined) await paratextPdp.setChapterUSX(new VerseRef('JHN', '1', '1'), verse);
paratextPdp.setExtensionData(
{ extensionName: 'foo', dataQualifier: 'fooData' },
'This is the data from extension foo',
Expand Down

0 comments on commit cd2923b

Please sign in to comment.