Support download timeout and ignoring failed imports options

@types/index.d.ts

@@ -63,6 +63,11 @@ export interface DbConfig {
    * A path to a SQLite database. Defaults to using an in-memory database.
    */
   sqlitePath?: string;
+
+  /**
+   * A better-sqlite3 database object. If provided, sqlitePath will be ignored.
+   */
+  db?: Database.Database;
 }
 
 export interface ExportConfig extends DbConfig, VerboseConfig {
@@ -114,6 +119,16 @@ export interface ImportConfig extends DbConfig, VerboseConfig {
    * Options passed to csv-parse for parsing GTFS CSV files.
    */
   csvOptions?: CsvParse.Options;
+
+  /**
+   * A timeout in milliseconds for downloading GTFS files. Defaults to no timeout.
+   */
+  downloadTimeout?: number;
+
+  /**
+   * Whether or not to ignore errors during import. Defaults to false.
+   */
+  ignoreErrors?: boolean;
 }
 
 export interface QueryOptions {

lib/db.js

@@ -17,7 +17,10 @@ function setupDb(sqlitePath) {
 export function openDb(config) {
   // If config is passed, use that to open or return db
   if (config) {
-    const { sqlitePath } = setDefaultConfig(config);
+    const { sqlitePath, db } = setDefaultConfig(config);
+    if (db) {
+      return db;
+    }
 
     if (dbs[sqlitePath]) {
       return dbs[sqlitePath];
@@ -34,7 +37,7 @@ export function openDb(config) {
 
   if (Object.keys(dbs).length > 1) {
     throw new Error(
-      'Multiple databases open, please specify which one to use.'
+      'Multiple databases open, please specify which one to use.',
     );
   }
 
@@ -44,14 +47,14 @@ export function openDb(config) {
 export function closeDb(db) {
   if (Object.keys(dbs).length === 0) {
     throw new Error(
-      'No database connection. Call `openDb(config)` before using any methods.'
+      'No database connection. Call `openDb(config)` before using any methods.',
     );
   }
 
   if (!db) {
     if (Object.keys(dbs).length > 1) {
       throw new Error(
-        'Multiple database connections. Pass the db you want to close as a parameter to `closeDb`.'
+        'Multiple database connections. Pass the db you want to close as a parameter to `closeDb`.',
       );
     }
 

lib/import.js

@@ -36,6 +36,9 @@ const downloadFiles = async (task) => {
   const response = await fetch(task.agency_url, {
     method: 'GET',
     headers: task.headers || {},
+    signal: task.downloadTimeout
+      ? AbortSignal.timeout(task.downloadTimeout)
+      : undefined,
   });
 
   if (response.status !== 200) {
@@ -594,8 +597,12 @@ const importFiles = (task) =>
         });
 
         parser.on('end', () => {
-          // Insert all remaining lines
-          importLines(task, lines, model, totalLineCount);
+          try {
+            // Insert all remaining lines
+            importLines(task, lines, model, totalLineCount);
+          } catch (error) {
+            reject(error);
+          }
           resolve();
         });
 
@@ -635,6 +642,7 @@ export async function importGtfs(initialConfig) {
         realtime_headers: agency.realtimeHeaders || false,
         realtime_urls: agency.realtimeUrls || false,
         downloadDir: path,
+        downloadTimeout: config.downloadTimeout,
         path: agency.path,
         csvOptions: config.csvOptions || {},
         ignoreDuplicates: config.ignoreDuplicates,
@@ -645,18 +653,26 @@ export async function importGtfs(initialConfig) {
         error: logError,
       };
 
-      if (task.agency_url) {
-        await downloadFiles(task);
-      }
+      try {
+        if (task.agency_url) {
+          await downloadFiles(task);
+        }
 
-      await readFiles(task);
-      await importFiles(task);
+        await readFiles(task);
+        await importFiles(task);
 
-      if (task.realtime_urls) {
-        await updateRealtimeData(task);
-      }
+        if (task.realtime_urls) {
+          await updateRealtimeData(task);
+        }
 
-      cleanup();
+        cleanup();
+      } catch (error) {
+        if (config.ignoreErrors) {
+          logError(error.message);
+        } else {
+          throw error;
+        }
+      }
     });
 
     log(