Add a service to lookup projects on the file system

extensions/src/hello-someone/hello-someone.ts

@@ -308,20 +308,6 @@ export async function activate(context: ExecutionActivationContext): Promise<voi
     peopleWebViewId || '',
   );
 
-  // Check that the project data provider from hello-world is working
-  const customProjectDataProvider =
-    await papi.projectDataProvider.getProjectDataProvider<'MyExtensionProjectTypeName'>(
-      'abcd',
-      'MyExtensionProjectTypeName',
-      'abc',
-    );
-  if ((await customProjectDataProvider.getMyProjectData('something')) !== 'my project data')
-    logger.error('Getting MyProjectData did not return the expected data');
-  else logger.info('Getting MyProjectData worked as expected');
-  if (await customProjectDataProvider.setMyProjectData('something', 'something else'))
-    logger.error('Setting MyProjectData should have returned false');
-  else logger.info('Setting MyProjectData worked as expected');
-
   // Await the registration promises at the end so we don't hold everything else up
   context.registrations.add(
     await peopleDataProviderPromise,

extensions/src/hello-world/hello-world.ts

@@ -9,12 +9,6 @@ import type { PeopleDataProvider } from 'hello-someone';
 import type { IWebViewProvider } from 'shared/models/web-view-provider.model';
 import type PapiEventEmitter from 'shared/models/papi-event-emitter.model';
 import type { HelloWorldEvent } from 'hello-world';
-import type {
-  ProjectDataProviderEngineFactory,
-  ProjectDataProviderEngineTypes,
-} from 'shared/models/project-data-provider-engine.model';
-import type { DataProviderUpdateInstructions } from 'shared/models/data-provider.model';
-import type { MyProjectDataType } from 'papi-shared-types';
 import helloWorldReactWebView from './web-views/hello-world.web-view?inline';
 import helloWorldReactWebViewStyles from './web-views/hello-world.web-view.scss?inline';
 import helloWorldReactWebView2 from './web-views/hello-world-2.web-view?inline';
@@ -160,30 +154,6 @@ export async function activate(context: ExecutionActivationContext): Promise<voi
     context.registrations.add(unsubGreetings);
   }
 
-  const projectDataProviderEngine: ProjectDataProviderEngineFactory<'MyExtensionProjectTypeName'> =
-    {
-      createProjectDataProviderEngine(
-        projectId: string,
-        projectStorageInterpreterId: string,
-      ): ProjectDataProviderEngineTypes['MyExtensionProjectTypeName'] {
-        logger.debug(`Creating PDP Engine for ${projectId}, ${projectStorageInterpreterId}`);
-        return {
-          getExtensionData: async (): Promise<string> => Promise.resolve('extension data'),
-          setExtensionData: async (): Promise<DataProviderUpdateInstructions<MyProjectDataType>> =>
-            Promise.resolve(false),
-          getMyProjectData: async (): Promise<string> => Promise.resolve('my project data'),
-          setMyProjectData: async (): Promise<DataProviderUpdateInstructions<MyProjectDataType>> =>
-            Promise.resolve(false),
-        };
-      },
-    };
-
-  const projectDataProviderEngineFactoryPromise =
-    papi.projectDataProvider.registerProjectDataProviderEngineFactory(
-      'MyExtensionProjectTypeName',
-      projectDataProviderEngine,
-    );
-
   // Await the registration promises at the end so we don't hold everything else up
   context.registrations.add(
     await htmlWebViewProviderPromise,
@@ -192,7 +162,6 @@ export async function activate(context: ExecutionActivationContext): Promise<voi
     onHelloWorldEmitter,
     await helloWorldPromise,
     await helloExceptionPromise,
-    await projectDataProviderEngineFactoryPromise,
   );
 
   logger.info('Hello World is finished activating!');

lib/papi-dts/papi.d.ts

@@ -2314,6 +2314,54 @@ declare module 'shared/services/data-provider.service' {
   const dataProviderService: DataProviderService;
   export default dataProviderService;
 }
+declare module 'shared/models/project-metadata.model' {
+  /**
+   * Low-level information describing a project that Platform.Bible directly manages and uses to load project data
+   */
+  export type ProjectMetadata = {
+    /**
+     * ID of the project (must be unique and case insensitive)
+     */
+    id: string;
+    /**
+     * Short name of the project (not necessarily unique)
+     */
+    name: string;
+    /**
+     * Indicates how the project is persisted to storage
+     */
+    storageType: string;
+    /**
+     * Indicates what sort of project this is which implies its data shape (e.g., what data streams should be available)
+     */
+    projectType: string;
+  };
+}
+declare module 'shared/models/project-lookup.model' {
+  import { ProjectMetadata } from 'shared/models/project-metadata.model';
+  /**
+   * Provides metadata for projects known by the platform
+   */
+  export interface ProjectLookupServiceType {
+    /**
+     * Provide metadata for all projects found on the local system
+     * @returns ProjectMetadata for all projects stored on the local system
+     */
+    getMetadataForAllProjects: () => Promise<ProjectMetadata[]>;
+    /**
+     * Look up metadata for a specific project ID
+     * @param projectId ID of the project to load
+     * @returns ProjectMetadata from the 'meta.json' file for the given project
+     */
+    getMetadataForProject: (projectId: string) => Promise<ProjectMetadata>;
+  }
+  export const ProjectLookupServiceNetworkObjectName = 'ProjectLookupService';
+}
+declare module 'shared/services/project-lookup.service' {
+  import { ProjectLookupServiceType } from 'shared/models/project-lookup.model';
+  const ProjectLookupService: ProjectLookupServiceType;
+  export default ProjectLookupService;
+}
 declare module 'shared/models/project-data-provider-engine.model' {
   import { ProjectTypes, ProjectDataTypes } from 'papi-shared-types';
   import type IDataProvider from 'shared/models/data-provider.interface';
@@ -2376,16 +2424,15 @@ declare module 'shared/services/project-data-provider.service' {
     pdpEngineFactory: ProjectDataProviderEngineFactory<ProjectType>,
   ): Promise<Dispose>;
   /**
-   * Get a Project Data Provider for the given project details.
-   * @param projectId ID for the project to load
-   * @param projectType Type of the project referenced by the given ID
-   * @param storageType Storage type of the project referenced by the given ID
-   * @returns Data provider with types that are associated with the given project type
-   */
+     * Get a Project Data Provider for the given project ID.
+     * For types to work properly, specify the project type as a generic parameter.
+     * @param projectId ID for the project to load
+     * @returns Data provider with types that are associated with the given project type
+     * @example const pdp = await getProjectDataProvider<'ParatextStandard'>('ProjectID12345');
+                pdp.getVerse(new VerseRef('JHN', '1', '1'));
+     */
   export function getProjectDataProvider<ProjectType extends ProjectTypes>(
     projectId: string,
-    projectType: ProjectType,
-    storageType: string,
   ): Promise<ProjectDataProvider[ProjectType]>;
   export interface PapiBackendProjectDataProviderService {
     registerProjectDataProviderEngineFactory: typeof registerProjectDataProviderEngineFactory;
@@ -2699,6 +2746,7 @@ declare module 'papi-frontend' {
   import { PapiWebViewService } from 'shared/services/web-view.service';
   import { InternetService } from 'shared/services/internet.service';
   import { DataProviderService } from 'shared/services/data-provider.service';
+  import { ProjectLookupServiceType } from 'shared/models/project-lookup.model';
   import { PapiFrontendProjectDataProviderService } from 'shared/services/project-data-provider.service';
   import { PapiContext } from 'renderer/context/papi-context/index';
   import { PapiHooks } from 'renderer/hooks/papi-hooks/index';
@@ -2751,6 +2799,10 @@ declare module 'papi-frontend' {
      * Service that gets project data providers
      */
     projectDataProvider: PapiFrontendProjectDataProviderService;
+    /**
+     * Provides metadata for projects known by the platform
+     */
+    projectLookup: ProjectLookupServiceType;
     react: {
       /**
        * All React contexts to be exposed on the papi
@@ -3024,6 +3076,7 @@ declare module 'papi-backend' {
   import { DataProviderService } from 'shared/services/data-provider.service';
   import { PapiBackendProjectDataProviderService } from 'shared/services/project-data-provider.service';
   import { ExtensionStorageService } from 'extension-host/services/extension-storage.service';
+  import { ProjectLookupServiceType } from 'shared/models/project-lookup.model';
   const papi: {
     /**
      * Event manager - accepts subscriptions to an event and runs the subscription callbacks when the event is emitted
@@ -3076,6 +3129,10 @@ declare module 'papi-backend' {
      * Service that registers and gets project data providers
      */
     projectDataProvider: PapiBackendProjectDataProviderService;
+    /**
+     * Provides metadata for projects known by the platform
+     */
+    projectLookup: ProjectLookupServiceType;
     /**
      * This service provides extensions in the extension host the ability to read/write data
      * based on the extension identity and current user (as identified by the OS). This service will

src/extension-host/extension-host.ts

@@ -9,6 +9,7 @@ import dataProviderService from '@shared/services/data-provider.service';
 import extensionAssetService from '@shared/services/extension-asset.service';
 import { getErrorMessage } from '@shared/utils/util';
 import { CommandNames } from 'papi-shared-types';
+import { startProjectLookupService } from '@extension-host/services/project-lookup-backend.service';
 
 // #region Test logs
 
@@ -53,6 +54,9 @@ networkService
     // The extension host is the only one that can initialize the extensionAssetService
     await extensionAssetService.initialize();
 
+    // Make sure project lookups are available before extensions look for them on PAPI
+    await startProjectLookupService();
+
     // The extension service locks down importing other modules, so be careful what runs after it
     await extensionService.initialize();
 

src/extension-host/services/papi-backend.service.ts

@@ -23,6 +23,8 @@ import {
 import extensionStorageService, {
   ExtensionStorageService,
 } from '@extension-host/services/extension-storage.service';
+import { ProjectLookupServiceType } from '@shared/models/project-lookup.model';
+import projectLookupService from '@shared/services/project-lookup.service';
 
 // IMPORTANT NOTES:
 // 1) When adding new services here, consider whether they also belong in papi-frontend.service.ts.
@@ -59,6 +61,8 @@ const papi = {
   /** JSDOC DESTINATION papiBackendProjectDataProviderService */
   projectDataProvider:
     papiBackendProjectDataProviderService as PapiBackendProjectDataProviderService,
+  /** JSDOC DESTINATION projectLookupService */
+  projectLookup: projectLookupService as ProjectLookupServiceType,
   /** JSDOC DESTINATION extensionStorageService */
   storage: extensionStorageService as ExtensionStorageService,
 };

src/extension-host/services/project-lookup-backend.service.ts

@@ -0,0 +1,135 @@
+import os from 'os';
+import path from 'path';
+import { ProjectMetadata } from '@shared/models/project-metadata.model';
+import {
+  ProjectLookupServiceNetworkObjectName,
+  ProjectLookupServiceType,
+} from '@shared/models/project-lookup.model';
+import { joinUriPaths } from '@node/utils/util';
+import logger from '@shared/services/logger.service';
+import networkObjectService from '@shared/services/network-object.service';
+import * as nodeFS from '@node/services/node-file-system.service';
+
+const PROJECTS_ROOT_URI = joinUriPaths('file://', os.homedir(), '.platform.bible', 'projects');
+const METADATA_FILE = 'meta.json';
+
+/** Get URIs to all projects stored locally on the file system */
+async function getProjectUris(): Promise<string[]> {
+  // Get all the directories in the projects root that match "<name>_<id>"
+  const entries = await nodeFS.readDir(PROJECTS_ROOT_URI, (entry) => {
+    return /^\w+_[^\W_]+$/.test(path.parse(entry).name);
+  });
+
+  return entries.directory;
+}
+
+/** Convert the contents of a 'meta.json' file to an object, ensuring its project ID is what we expect */
+function convertToMetadata(jsonString: string, expectedID: string): ProjectMetadata {
+  const md: ProjectMetadata = JSON.parse(jsonString);
+  if ('id' in md && 'name' in md && 'storageType' in md && 'projectType' in md) {
+    if (md.id.toUpperCase() !== expectedID.toUpperCase()) throw new Error('Mismatched IDs');
+    return md;
+  }
+  throw new Error(`Invalid ProjectMetadata JSON: ${jsonString}`);
+}
+
+/** Load the contents of all 'meta.json' files from disk */
+async function loadAllProjectsMetadata(): Promise<Set<ProjectMetadata>> {
+  const retVal = new Set<ProjectMetadata>();
+  const uris = await getProjectUris();
+
+  await Promise.all(
+    uris.map(async (uri) => {
+      try {
+        const metadataString = await nodeFS.readFileText(joinUriPaths(uri, METADATA_FILE));
+        const projectID = uri.substring(uri.lastIndexOf('_') + 1);
+        retVal.add(convertToMetadata(metadataString, projectID));
+      } catch (error) {
+        logger.warn(`Skipping project directory with missing/invalid metadata file: ${uri}`);
+      }
+    }),
+  );
+
+  return retVal;
+}
+
+/** Lookup or load the contents of a 'meta.json' file from disk for a given project ID */
+async function getProjectMetadata(projectId: string): Promise<ProjectMetadata> {
+  const idUpper = projectId.toUpperCase();
+  const uris = await getProjectUris();
+  const matches = uris.filter((uri) => uri.toUpperCase().endsWith(`_${idUpper}`));
+  if (matches.length === 0) throw new Error(`No known project with ID ${projectId}`);
+  if (matches.length > 1) throw new Error(`${matches.length} projects share the ID ${projectId}`);
+
+  const metadataPath = joinUriPaths(matches[0], METADATA_FILE);
+  const metadataString = await nodeFS.readFileText(metadataPath);
+  return convertToMetadata(metadataString, projectId);
+}
+
+// Map of project ID to 'meta.json' contents for that project
+const localProjects = new Map<string, ProjectMetadata>();
+
+/** Refresh the map of all project 'meta.json' data */
+async function reloadMetadata(): Promise<void> {
+  const allMetadata = await loadAllProjectsMetadata();
+  localProjects.clear();
+  allMetadata.forEach((metadata) => {
+    localProjects.set(metadata.id.toUpperCase(), metadata);
+  });
+}
+
+let initializationPromise: Promise<void>;
+async function initialize(): Promise<void> {
+  if (!initializationPromise) {
+    initializationPromise = new Promise<void>((resolve, reject) => {
+      const executor = async () => {
+        try {
+          await reloadMetadata();
+          resolve();
+        } catch (error) {
+          reject(error);
+        }
+      };
+      executor();
+    });
+  }
+  return initializationPromise;
+}
+
+async function getMetadataForAllProjects(): Promise<ProjectMetadata[]> {
+  await initialize();
+  return [...localProjects.values()];
+}
+
+async function getMetadataForProject(projectId: string): Promise<ProjectMetadata> {
+  await initialize();
+  const existingValue = localProjects.get(projectId);
+  if (existingValue) return existingValue;
+
+  // Try to load the project directly in case the files were copied after initialization
+  const newMetadata = await getProjectMetadata(projectId);
+  localProjects.set(newMetadata.id, newMetadata);
+  return newMetadata;
+}
+
+const ProjectLookupService: ProjectLookupServiceType = {
+  getMetadataForAllProjects,
+  getMetadataForProject,
+};
+
+let networkObject: ProjectLookupServiceType;
+
+/**
+ * Register the network object that backs the PAPI project lookup service
+ */
+export async function startProjectLookupService(): Promise<void> {
+  await initialize();
+  networkObject = await networkObjectService.set<ProjectLookupServiceType>(
+    ProjectLookupServiceNetworkObjectName,
+    ProjectLookupService,
+  );
+}
+
+export function getNetworkObject(): ProjectLookupServiceType {
+  return networkObject;
+}

src/main/main.ts

@@ -18,7 +18,6 @@ import * as networkService from '@shared/services/network.service';
 import * as commandService from '@shared/services/command.service';
 import { resolveHtmlPath } from '@node/utils/util';
 import extensionHostService from '@main/services/extension-host.service';
-import * as pdpFactoryService from '@main/services/project-data-provider-factory.service';
 import networkObjectService from '@shared/services/network-object.service';
 import extensionAssetProtocolService from '@main/services/extension-asset-protocol.service';
 import { wait } from '@shared/utils/util';
@@ -268,16 +267,6 @@ async function main() {
 
   // #endregion
 
-  // #region Setup and test project data providers
-
-  // This won't be needed long-term if we don't have built-in project data provider factories
-  // Maybe all of this will live in extensions, even for built-in project types. Who knows?
-  await pdpFactoryService.initialize();
-
-  await pdpFactoryService.test();
-
-  // #endregion
-
   // #region Test network objects
 
   const testMain = {