diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d6ce07a04..68d6d76164 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1008,7 +1008,9 @@ IF(MSVC) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c DESTINATION ${netCDF_BINARY_DIR}/ncdump/) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c - DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c + DESTINATION ${netCDF_BINARY_DIR}/ncdap_test/) ENDIF() ENDIF() diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index bdda0e49fc..97eaa684db 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.1 - TBD +* [Bug Fix] Re-enable DAP2 authorization testing. See [Github #2011](https://github.com/Unidata/netcdf-c/issues/2011). * [Bug Fix] Fix bug with windows version of mkstemp that causes failure to create more than 26 temp files. See [Github #1998](https://github.com/Unidata/netcdf-c/pull/1998). * [Bug Fix] Fix ncdump bug when printing VLENs with basetype char. See [Github #1986](https://github.com/Unidata/netcdf-c/issues/1986). * [Bug Fixes] The netcdf-c library was incorrectly determining the scope of types referred to by nc_inq_type_equal. See [Github #1959](https://github.com/Unidata/netcdf-c/pull/1959) for more information. diff --git a/libdap2/cdf.c b/libdap2/cdf.c index cc5f522b18..4349dbe687 100644 --- a/libdap2/cdf.c +++ b/libdap2/cdf.c @@ -1089,6 +1089,7 @@ free1cdfnode(CDFnode* node) nullfree(node->ocname); nullfree(node->ncbasename); nullfree(node->ncfullname); + nullfree(node->dodsspecial.dimname); if(node->attributes != NULL) { for(j=0;jattributes);j++) { NCattribute* att = (NCattribute*)nclistget(node->attributes,j); @@ -1099,7 +1100,6 @@ free1cdfnode(CDFnode* node) nullfree(att); } } - nullfree(node->dodsspecial.dimname); nclistfree(node->subnodes); nclistfree(node->attributes); nclistfree(node->array.dimsetplus); diff --git a/libdap2/dapattr.c b/libdap2/dapattr.c index f0dedfb8e4..c0e37aa9a7 100644 --- a/libdap2/dapattr.c +++ b/libdap2/dapattr.c @@ -79,6 +79,8 @@ fprintf(stderr,"%s.maxstrlen=%d\n",node->ocname,(int)node->dodsspecial.maxstrlen #endif } else if(strcmp(ocname,"DODS.dimName")==0 || strcmp(ocname,"DODS_EXTRA.dimName")==0) { + nullfree(node->dodsspecial.dimname); /* in case repeated */ + node->dodsspecial.dimname = NULL; if(values != NULL) { nullfree(node->dodsspecial.dimname); node->dodsspecial.dimname = nulldup(values[0]); diff --git a/libdap2/ncd2dispatch.c b/libdap2/ncd2dispatch.c index 9ecb0932d7..d34ee4fa46 100644 --- a/libdap2/ncd2dispatch.c +++ b/libdap2/ncd2dispatch.c @@ -403,9 +403,8 @@ fprintf(stderr,"ce=%s\n",dumpconstraint(dapcomm->oc.dapconstraint)); ocstat = oc_open(dapcomm->oc.urltext,&dapcomm->oc.conn); if(ocstat != OC_NOERR) {THROWCHK(ocstat); goto done;} -#ifdef DEBUG1 - (void)oc_trace_curl(dapcomm->oc.conn); -#endif + if(getenv("CURLOPT_VERBOSE") != NULL) + (void)oc_trace_curl(dapcomm->oc.conn); nullfree(dapcomm->oc.urltext); /* clean up */ dapcomm->oc.urltext = NULL; diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index 1fdde93f15..691ec28205 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -71,8 +71,10 @@ set_curlflag(NCD4INFO* state, int flag) break; case CURLOPT_NETRC: case CURLOPT_NETRC_FILE: if(state->auth->curlflags.netrc) { - SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED); - SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); + SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_OPTIONAL); + /* IF HTTP.NETRC is set with "", then assume the default .netrc file (which is apparently CWD) */ + if(strlen(state->auth->curlflags.netrc)>0) + SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); } break; case CURLOPT_VERBOSE: diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 47a460562b..cb24c0b9f1 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -117,6 +117,11 @@ NCD4_open(const char * path, int mode, ncsetlogging(1); } + /* Check env values */ + if(getenv("CURLOPT_VERBOSE") != NULL) + d4info->auth->curlflags.verbose = 1; + + /* Setup a curl connection */ { CURL* curl = NULL; /* curl handle*/ diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 8075b7373b..31c34a9321 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -35,7 +35,7 @@ tst_h_scalar tst_rename tst_rename2 tst_rename3 tst_h5_endians \ tst_atts_string_rewrite tst_hdf5_file_compat tst_fill_attr_vanish \ tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \ tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \ -tst_bug1442 tst_charvlenbug +tst_bug1442 # Temporary I hoped, but hoped in vain. if !ISCYGWIN @@ -43,7 +43,7 @@ NC4_TESTS += tst_h_strbug tst_h_refs endif # Build test programs plus programs used in test scripts. -check_PROGRAMS = $(NC4_TESTS) tst_empty_vlen_unlim +check_PROGRAMS = $(NC4_TESTS) tst_empty_vlen_unlim tst_charvlenbug TESTS = $(NC4_TESTS) run_empty_vlen_test.sh # Add these if large file tests are turned on. diff --git a/nc_test4/tst_misc.sh b/nc_test4/tst_misc.sh index 1e594d3fdf..767c4f7a17 100755 --- a/nc_test4/tst_misc.sh +++ b/nc_test4/tst_misc.sh @@ -16,6 +16,7 @@ echo "*** Fail: phony dimension creation" ECODE=1 fi +if "x$NC_VLEN_NOTEST" = x1 ; then echo "*** Testing char(*) type printout error in ncdump" rm -f ./tst_charvlenbug.nc ./tmp ${execdir}/tst_charvlenbug @@ -25,6 +26,7 @@ else echo "*** Fail: char(*) ncdump printout" ECODE=1 fi +fi rm -f tmp diff --git a/nc_test4/tst_szip.sh b/nc_test4/tst_szip.sh index dfffdb0cab..f2d48d05f1 100755 --- a/nc_test4/tst_szip.sh +++ b/nc_test4/tst_szip.sh @@ -8,7 +8,7 @@ set -e rm -f testnc.h5 testszip.nc szip_dump.cdl echo "*** Test read of known szip file" -${NCDUMP} ${srcdir}/ref_szip.h5 >szip_dump.cdl +${NCDUMP} -n ref_szip ${srcdir}/ref_szip.h5 >szip_dump.cdl diff -w ${srcdir}/ref_szip.cdl ./szip_dump.cdl echo "*** Testing tst_szip " diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index 6b0659ad1a..8e55b6c041 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -14,19 +14,28 @@ FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) -IF(ENABLE_DAP_REMOTE_TESTS) +IF(ENABLE_TESTS) + + IF(ENABLE_DAP_REMOTE_TESTS) BUILD_BIN_TEST(findtestserver) BUILD_BIN_TEST(pingurl) -ENDIF() + ENDIF() + + IF(USE_X_GETOPT) + SET(XGSRC XGetopt.c) + ENDIF() + BUILD_BIN_TEST(pathcvt ${XGSRC}) -IF(ENABLE_TESTS) # Base tests # The tests are set up as a combination of shell scripts and executables that # must be run in a particular order. It is painful but will use macros to help # keep it from being too bad. # Binary Test Macro - add_sh_test(ncdap tst_ncdap3) + IF(BUILD_UTILITIES) + add_sh_test(ncdap tst_ncdap3) + add_sh_test(ncdap testpathcvt) + ENDIF() IF(NOT MSVC) add_bin_env_test(ncdap t_dap3a) diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index e25a73b963..d5c514a0ff 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -32,15 +32,17 @@ check_PROGRAMS += t_dap3a test_cvt3 test_vara TESTS += t_dap3a test_cvt3 test_vara if BUILD_UTILITIES TESTS += tst_ncdap3.sh +TESTS += testpathcvt.sh endif # remote tests are optional # because the server may be down or inaccessible if ENABLE_DAP_REMOTE_TESTS -noinst_PROGRAMS = findtestserver pingurl +noinst_PROGRAMS = findtestserver pingurl pathcvt findtestserver_SOURCES = findtestserver.c pingurl_SOURCES = pingurl.c +pathcvt_SOURCES = pathcvt.c endif if ENABLE_DAP_REMOTE_TESTS @@ -91,10 +93,10 @@ EXTRA_DIST = tst_ncdap3.sh \ tst_zero_len_var.sh \ tst_filelists.sh tst_urls.sh tst_utils.sh \ t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \ - t_ncf330.c tst_ber.sh tst_fillmismatch.sh tst_encode.sh \ - findtestserver.c.in + t_ncf330.c tst_ber.sh tst_fillmismatch.sh tst_encode.sh testpathcvt.sh \ + findtestserver.c.in ref_pathcvt.txt -CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe +CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe tmp*.txt # This should only be left behind if using parallel io CLEANFILES += tmp_* diff --git a/ncdap_test/pathcvt.c b/ncdap_test/pathcvt.c new file mode 100755 index 0000000000..d85e0564f6 --- /dev/null +++ b/ncdap_test/pathcvt.c @@ -0,0 +1,137 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "ncpathmgr.h" + +/* +Synopsis + +pathcvt [-u|-w|-m|-c] [-e] PATH + +Options + -e add backslash escapes to '\' and ' ' +Output type options: + -u convert to Unix form of path + -w convert to Windows form of path + -m convert to MSYS form of path + -c convert to Cygwin form of path + +Default is to convert to the format used by the platform. + +*/ + +#define DEBUG + +struct Options { + int target; + int escape; + int debug; +} cvtoptions; + +static char* escape(const char* path); +static void usage(const char* msg); + +int +main(int argc, char** argv) +{ + int c; + char* cvtpath = NULL; + char* inpath; + + memset((void*)&cvtoptions,0,sizeof(cvtoptions)); + + while ((c = getopt(argc, argv, "cD:ehmuw")) != EOF) { + switch(c) { + case 'c': cvtoptions.target = NCPD_CYGWIN; break; + case 'e': cvtoptions.escape = 1; break; + case 'h': usage(NULL); break; + case 'm': cvtoptions.target = NCPD_MSYS; break; + case 'u': cvtoptions.target = NCPD_NIX; break; + case 'w': cvtoptions.target = NCPD_WIN; break; + case 'D': + sscanf(optarg,"%d",&cvtoptions.debug); + break; + case '?': + usage("unknown option"); + break; + } + } + + argc -= optind; + argv += optind; + + /* If no file arguments left or more than one, print usage message. */ + if (argc == 0) + usage("no path specified"); + if (argc > 1) + usage("more than one path specified"); + inpath = argv[0]; + if(cvtoptions.target == NCPD_UNKNOWN) + cvtpath = NCpathcvt(inpath); + else + cvtpath = NCpathcvt_test(inpath,cvtoptions.target,'c'); + if(cvtpath && cvtoptions.escape) { + char* path = cvtpath; cvtpath = NULL; + cvtpath = escape(path); + free(path); + } + printf("%s",cvtpath); + if(cvtpath) free(cvtpath); + return 0; +} + +static void +usage(const char* msg) +{ + if(msg != NULL) fprintf(stderr,"%s\n",msg); + fprintf(stderr,"pathcvt [-u|-w|-m|-c] PATH\n"); + if(msg == NULL) exit(0); else exit(1); +} + +static char* +escape(const char* path) +{ + size_t slen = strlen(path); + const char* p; + char* q; + char* epath = NULL; + + epath = (char*)malloc((2*slen) + 1); + if(epath == NULL) usage("out of memtory"); + p = path; + q = epath; + for(;*p;p++) { + switch (*p) { + case '\\': case ' ': + *q++ = '\\'; + /* fall thru */ + default: + *q++ = *p; + break; + } + } + *q = '\0'; + return epath; +} + diff --git a/ncdap_test/ref_pathcvt.txt b/ncdap_test/ref_pathcvt.txt new file mode 100644 index 0000000000..1398eb14f9 --- /dev/null +++ b/ncdap_test/ref_pathcvt.txt @@ -0,0 +1,45 @@ +path: /xxx/a/b + /xxx/a/b + /cygdrive/c/xxx/a/b + /c/xxx/a/b + c:\\xxx\\a\\b +path: d:/x/y + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\\x\\y +path: /cygdrive/d/x/y + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\\x\\y +path: /d/x/y + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\\x\\y +path: /cygdrive/d + /d + /cygdrive/d + /d + d: +path: /d + /d + /cygdrive/d + /d + d: +path: /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + /d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + /d/git/netcdf-c/dap4_test/test_anon_dim.2.syn + d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn +path: d:\x\y + /d/x/y + /cygdrive/d/x/y + /d/x/y + d:\\x\\y +path: d:\x\y w\z + /d/x/y w/z + /cygdrive/d/x/y w/z + /d/x/y w/z + d:\\x\\y\\ w\\z diff --git a/ncdap_test/t_auth.c b/ncdap_test/t_auth.c index 2156154afe..e70a068e87 100644 --- a/ncdap_test/t_auth.c +++ b/ncdap_test/t_auth.c @@ -28,15 +28,17 @@ See \ref copyright file for more info. #define KEEPRC -#define RC ".ocrc" -#define SPECRC "./ocrc" +#define AUTHTESTSERVER "thredds.ucar.edu" -#define USERPWD "tiggeUser:tigge" +#define RC ".daprc" +#define SPECRC "./daprc" + +#define USERPWD "authtester:auth" #define COOKIEFILE "./cookies" -#define URL1 "https://%s@%s/dodsC/restrict/testData.nc" -#define URL2 "https://%s/dodsC/restrict/testData.nc" -#define URL3 "https://%s@thredds-test.ucar.edu/thredds/dodsC/restrict/testData.nc" +#define URL1 "https://%s@%s/thredds/dodsC/test3/testData.nc" +#define URL2 "https://thredds/%s/dodsC/test3/testData.nc" +#define URL3 "https://%s@" AUTHTESTSERVER "/thredds/dodsC/test3/testData.nc" /* Embedded user:pwd */ static char url1[1024]; @@ -45,36 +47,25 @@ static char url1[1024]; static char url2[1024]; /* Test redirect from different machine*/ +#ifndef NOREDIR static char url3[1024]; +#endif static int testrc(const char* prefix, const char* url); static void fillrc(const char* path); static void killrc(); -#ifdef DEBUG -static void -CHECK(int e, const char* msg) -{ - if(e == NC_NOERR) return; - if(msg == NULL) msg = "Error"; - fprintf(stderr,"%s: %s\n", msg, nc_strerror(e)); - exit(1); -} -#endif - int main(int argc, char** argv) { int ncid,retval,pass; - FILE* rc; const char* dfaltsvc; - char buffer[8192]; const char* home; fprintf(stderr,"Testing: Authorization\n"); - dfaltsvc = nc_findtestserver("thredds",REMOTETESTSERVERS); - if(svc == NULL) { + dfaltsvc = nc_findtestserver("thredds",AUTHTESTSERVER); + if(dfaltsvc == NULL) { fprintf(stderr,"WARNING: Cannot locate test server\n"); exit(0); } @@ -110,7 +101,7 @@ fflush(stderr); #ifndef NOLOCAL { /* Test 1: RC in ./ */ - fprintf(stderr,"Testing: user:pwd in %s/%s: %s\n",".",RC); + fprintf(stderr,"Testing: user:pwd in %s/%s\n",".",RC); if(!testrc(".",url2)) { fprintf(stderr,"user:pwd in %s/%s failed\n",".",RC); exit(1); @@ -122,7 +113,7 @@ fflush(stderr); { /* Test 1: RC in HOME */ home = getenv("HOME"); - fprintf(stderr,"user:pwd in %s/%s: %s\n",home,RC); + fprintf(stderr,"user:pwd in %s/%s\n",home,RC); if(!testrc(home,url2)) { fprintf(stderr,"user:pwd in %s/%s failed\n",home,RC); exit(1); @@ -205,10 +196,10 @@ static void killrc() { const char* home; - char path[1024]; #ifdef KEEPRC fprintf(stderr,"kill: ./%s\n",RC); -#else +#else + char path[1024]; snprintf(path,sizeof(path),"%s/%s",".",RC); unlink(path); /* delete the file */ #endif diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 8a16661fc9..c35dd81530 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -39,9 +39,14 @@ fi LOCALRCFILES="$WD/.dodsrc $WD/.daprc $WD/.ncrc $WD/$NETRC $WD/$NETRCIMP" HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP" - NETRCFILE=$WD/$NETRC DAPRCFILE=$WD/$RC +if test "x$FP_ISMSVC" = x1 ; then + LOCALRCFILES=`${execdir}/pathcvt "$LOCALRCFILES"` + HOMERCFILES=`${execdir}/pathcvt "$HOMERCFILES"` + NETRCFILE=`${execdir}/pathcvt "$NETRCFILE"` + DAPRCFILE=`${execdir}/pathcvt "$DAPRCFILE"` +fi HOMENETRCFILE=$HOME/$NETRC HOMEDAPRCFILE=$HOME/$RC @@ -137,9 +142,9 @@ rclocal1() { # Case: local daprc local netrc no embed rclocal2() { - echo "***Testing rc file in local directory" + echo "***Testing rc file + .netrc in local directory" reset - # Create the rc file and (optional) netrc fil in ./ + # Create the rc file and (optional) netrc file in ./ createnetrc $LOCALNETRC createrc $LOCALRC $LOCALNETRC # Invoke ncdump to extract a file using the URL @@ -214,6 +219,5 @@ if test "x$RCHOME" = x1 ; then fi reset - exit diff --git a/ncdap_test/testpathcvt.sh b/ncdap_test/testpathcvt.sh new file mode 100755 index 0000000000..e4588b66a4 --- /dev/null +++ b/ncdap_test/testpathcvt.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +testcase1() { +T="$1" +P="$2" +C=`${execdir}/pathcvt $T "$P"` +echo " $C" +} + +testcase() { +#X=`echo -n "$1" | sed -e 's/\\\/\\\\\\\/g'` +X="$1" +echo "path:" "$X" + testcase1 "-u" "$1" + testcase1 "-c" "$1" + testcase1 "-m" "$1" + testcase1 "-ew" "$1" | sed -e 's/\\/\\\\/g' +} + +rm -f tmp_pathcvt.txt + +testcase "/xxx/a/b" >> tmp_pathcvt.txt +testcase "d:/x/y" >> tmp_pathcvt.txt +testcase "/cygdrive/d/x/y" >> tmp_pathcvt.txt +testcase "/d/x/y" >> tmp_pathcvt.txt +testcase "/cygdrive/d" >> tmp_pathcvt.txt +testcase "/d" >> tmp_pathcvt.txt +testcase "/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn" >> tmp_pathcvt.txt +testcase "d:\\x\\y" >> tmp_pathcvt.txt +testcase "d:\\x\\y w\\z" >> tmp_pathcvt.txt + +diff -w ${srcdir}/ref_pathcvt.txt ./tmp_pathcvt.txt + +exit 0 diff --git a/ncdap_test/testurl.sh b/ncdap_test/testurl.sh index 17ae9231e9..92c08cb2fc 100755 --- a/ncdap_test/testurl.sh +++ b/ncdap_test/testurl.sh @@ -1,8 +1,8 @@ #!/bin/sh -#NOP=1 -#NOS=1 -#NOB=1 +NOP=1 +NOS=1 +NOB=1 #SHOW=1 #DBG=1 diff --git a/oc2/occurlfunctions.c b/oc2/occurlfunctions.c index b6d7d3e47c..076ecfb625 100644 --- a/oc2/occurlfunctions.c +++ b/oc2/occurlfunctions.c @@ -66,8 +66,10 @@ ocset_curlflag(OCstate* state, int flag) case CURLOPT_NETRC: case CURLOPT_NETRC_FILE: if(state->auth->curlflags.netrc) { - SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_REQUIRED); - SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); + SETCURLOPT(state, CURLOPT_NETRC, (OPTARG)CURL_NETRC_OPTIONAL); + /* IF HTTP.NETRC is set with "", then assume the default .netrc file (which is apparently CWD) */ + if(strlen(state->auth->curlflags.netrc)>0) + SETCURLOPT(state, CURLOPT_NETRC_FILE, state->auth->curlflags.netrc); } break;