got async working with netcdf integration with one computational comp…

…onent

src/clib/pio_msg.c

@@ -28,6 +28,10 @@ extern int pio_log_level;
 extern int event_num[2][NUM_EVENTS];
 #endif /* USE_MPE */
 
+/* Prototypes from nc4internal.h. */
+int nc4_file_list_add(int ncid, const char *path, int mode,
+                      void **dispatchdata);
+
 /**
  * This function is run on the IO tasks to handle nc_inq_type*()
  * functions.
@@ -168,20 +172,14 @@ int set_fill_handler(iosystem_desc_t *ios)
  */
 int create_file_handler(iosystem_desc_t *ios)
 {
-<<<<<<< HEAD
-=======
     int ncid = 0;
->>>>>>> master
     int len;
     int iotype;
     int mode;
     int use_ext_ncid;
     char ncidp_present;
-<<<<<<< HEAD
-    int *ncidp = NULL;
-=======
->>>>>>> master
     int mpierr;
+    int ret;
 
     PLOG((1, "create_file_handler comproot = %d", ios->comproot));
     assert(ios);
@@ -212,8 +210,17 @@ int create_file_handler(iosystem_desc_t *ios)
           use_ext_ncid, ncidp_present, ncid));
 
     /* Call the create file function. */
-    PIOc_createfile_int(ios->iosysid, &ncid, &iotype, filename, mode,
-                        use_ext_ncid);
+    if (use_ext_ncid)
+    {
+        PLOG((2, "about to call nc_create()"));
+        nc_create(filename, mode|NC_UDF0, &ncid);
+    }
+    else
+    {
+        PLOG((2, "about to call PIOc_createfile_int()"));
+        PIOc_createfile_int(ios->iosysid, &ncid, &iotype, filename, mode,
+                            use_ext_ncid);
+    }
 
     PLOG((1, "create_file_handler succeeded!"));
     return PIO_NOERR;

src/clib/pioc.c

@@ -19,8 +19,12 @@ extern int event_num[2][NUM_EVENTS];
 #endif /* USE_MPE */
 
 #ifdef NETCDF_INTEGRATION
-/* Have we initialized? */
+/* Have we initialized the netcdf integration code? */
 extern int ncint_initialized;
+
+/* This is used as the default iosysid for the netcdf integration
+ * code. */
+extern int diosysid;
 #endif /* NETCDF_INTEGRATION */
 
 /**
@@ -1736,6 +1740,16 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
         /* Add this id to the list of PIO iosystem ids. */
         iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys);
         PLOG((2, "new iosys ID added to iosystem_list iosysidp[%d] = %d", cmp, iosysidp[cmp]));
+
+#ifdef NETCDF_INTEGRATION
+        if (in_io || in_cmp)
+        {
+            /* Remember the io system id. */
+            diosysid = iosysidp[cmp];
+            PLOG((3, "diosysid = %d", iosysidp[cmp]));
+        }
+#endif /* NETCDF_INTEGRATION */
+
     } /* next computational component */
 
     /* Now call the function from which the IO tasks will not return

src/clib/pioc_support.c

@@ -1935,8 +1935,12 @@ PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
  * @param iosysid A defined pio system ID, obtained from
  * PIOc_Init_Intracomm() or PIOc_InitAsync().
  * @param ncidp A pointer that gets the ncid of the newly created
- * file. For NetCDF integration, this contains the ncid assigned by
- * the netCDF layer, which is used instead of a PIO-generated ncid.
+ * file. This is the PIO ncid. Within PIO, the file will have a
+ * different ID, the file->fh. When netCDF integration is used, the
+ * PIO ncid is also stored in the netcdf-c internal file list, and the
+ * PIO code is called by the netcdf-c dispatch code. In this case,
+ * there are two ncids for the file, the PIO ncid, and the file->fh
+ * ncid.
  * @param iotype A pointer to a pio output format. Must be one of
  * PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, or
  * PIO_IOTYPE_NETCDF4P.
@@ -2003,7 +2007,6 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
         if (!ios->ioproc)
         {
             int msg = PIO_MSG_CREATE_FILE;
-            char ncidp_present = ncidp ? 1 : 0;
             size_t len = strlen(filename);
             char ncidp_present = ncidp ? 1 : 0;
 
@@ -2073,6 +2076,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
             break;
 #endif
         }
+        PLOG((3, "create call complete file->fh %d", file->fh));
     }
 
     /* Broadcast and check the return code. */
@@ -2117,6 +2121,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
         if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid)))
             return pio_err(NULL, file, ierr, __FILE__, __LINE__);
         file->pio_ncid = file->pio_ncid << ID_SHIFT;
+        PLOG((2, "changed ncid to file->pio_ncid = %d", file->pio_ncid));
     }
     PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid));
 

tests/ncint/tst_pio_async.c

@@ -54,6 +54,8 @@ main(int argc, char **argv)
 
         /* Turn on logging for PIO library. */
         PIOc_set_log_level(3);
+        if (!my_rank)
+            nc_set_log_level(3);
 
         /* Initialize the intracomm. */
         if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT,
@@ -63,7 +65,7 @@ main(int argc, char **argv)
         if (my_rank)
         {
             /* Create a file with a 3D record var. */
-            if (nc_create(FILE_NAME, NC_UDF0, &ncid)) PERR;
+            if (nc_create(FILE_NAME, NC_UDF0|NC_NETCDF4, &ncid)) PERR;
             /* if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; */
             /* if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; */
             /* if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; */

tests/ncint/tst_pio_udf.c

@@ -61,7 +61,7 @@ main(int argc, char **argv)
         int i;
 
         /* Turn on logging for PIO library. */
-        /* PIOc_set_log_level(3); */
+        PIOc_set_log_level(3);
 
         /* Initialize the intracomm. */
         if (nc_def_iosystemm(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR;