Merge branch 'no_last_slice'

doc/Notes.html

@@ -2830,7 +2830,18 @@ <h4>Cross reference matrix</h4>
               <td>1.3.0</td>
               <td>1.2.2</td>
 	    </tr>
-
+	    <tr>
+	      <td>June 6th, 2024</td>
+              <th>2.7.15</th>
+              <td>11.3</td>
+              <td>06</td>
+              <td>6.7.3</td>
+              <td>1.7.0</td>
+              <td>1.6.0</td>
+              <td>1.9.0</td>
+              <td>1.3.0</td>
+              <td>1.2.2</td>
+	    </tr>
 	  </table>
 	</div>
 

man/dar.1

@@ -1,4 +1,4 @@
-.TH DAR 1 "March 23rd, 2024"
+.TH DAR 1 "June 6th, 2024"
 .UC 8
 .SH NAME
 dar \- creates, tests, lists, extracts, compares, merges, isolates, repairs dar archives

misc/dar_static_builder_with_musl_voidlinux.bash

@@ -33,8 +33,8 @@ LIBSSH2_VERSION=1.10.0
 # define whether to use libssh or libssh2
 # note that both libraries do support ssh version 2 protocol,
 # libssh even supports more recent ciphers than libssh2
-# LIBSSH=1
-LIBSSH=2
+LIBSSH=1
+# LIBSSH=2
 
 
 # wget options need for gnutls website that does not provide all chain of trust in its certificate

src/build/ChangeLog

@@ -32,10 +32,14 @@ from 2.7.14 to 2.7.15
   reported as deprecated by recent libcurl libraries.
 - fixed compilation problem under MacOS Mojave
 - fixed bug that lead the warning about a backup operation about to save
-  the backup itslef, to not show
+  itself, to not show
 - removing obsolete call to gcry_control(GCRYCTL_ENABLE_M_GUARD) while
-  initializing libgcrypt. This lead to libgcrypt initialization to fail
+  initializing libgcrypt. This led to libgcrypt initialization to fail
   with libgcrypt 1.11 and more recent versions.
+- fixed listing but about "present but unsaved" FSA status
+- fixed dead-lock condition in libdar when used with libcurl > 0.74.0 at
+  the end of closing sftp session (undocumented changed behavior in
+  libcurl).
 
 from 2.7.13 to 2.7.14
 - adding safe guard in fichier_libcurl destructor to verify all data have

src/check/main.sh

@@ -266,7 +266,9 @@ listing:
 
 testing:
 
-diffing:
+diff:
+--fsa-scope none
+# because of birth date change when comparing with restored data
 
 EOF
 

src/check/routine.sh

@@ -625,19 +625,19 @@ fi
 # I1 - SFTP repository
 #
 if echo $* | grep "I1" > /dev/null ; then
-GO "I1-1" 0 $ROUTINE_DEBUG $DAR -N -w -c $DAR_SFTP_REPO/$prefix$full -R $src "-@" $prefix$catf_fly -B $OPT
-GO "I1-2" 0 $ROUTINE_DEBUG $DAR -N -l $DAR_SFTP_REPO/$prefix$full -B $OPT
-GO "I1-3" 0 $ROUTINE_DEBUG $DAR -N -t $DAR_SFTP_REPO/$prefix$full -B $OPT
+GO "I1-1" 0 $ROUTINE_DEBUG $DAR -N -w -c $DAR_SFTP_REPO/$prefix-$full -R $src "-@" $prefix$catf_fly -B $OPT
+GO "I1-2" 0 $ROUTINE_DEBUG $DAR -N -l $DAR_SFTP_REPO/$prefix-$full -B $OPT
+GO "I1-3" 0 $ROUTINE_DEBUG $DAR -N -t $DAR_SFTP_REPO/$prefix-$full -B $OPT
 ../sftp_mdelete "$DAR_SFTP_REPO" "$prefix*"
 fi
 
 #
 # I2 - FTP repository
 #
 if echo $* | grep "I2" > /dev/null ; then
-GO "I2-1" 0 $ROUTINE_DEBUG $DAR -N -w -c $DAR_FTP_REPO/$prefix$full -R $src "-@" $prefix$catf_fly -B $OPT
-GO "I2-2" 0 $ROUTINE_DEBUG $DAR -N -l $DAR_FTP_REPO/$prefix$full -B $OPT
-GO "I2-3" 0 $ROUTINE_DEBUG $DAR -N -t $DAR_FTP_REPO/$prefix$full -B $OPT
+GO "I2-1" 0 $ROUTINE_DEBUG $DAR -N -w -c $DAR_FTP_REPO/$prefix-$full -R $src "-@" $prefix$catf_fly -B $OPT
+GO "I2-2" 0 $ROUTINE_DEBUG $DAR -N -l $DAR_FTP_REPO/$prefix-$full -B $OPT
+GO "I2-3" 0 $ROUTINE_DEBUG $DAR -N -t $DAR_FTP_REPO/$prefix-$full -B $OPT
 ../ftp_mdelete "$DAR_FTP_REPO" "$prefix*"
 fi
 ###

src/libdar/cat_entree.cpp

@@ -369,14 +369,19 @@ namespace libdar
 		}
 	    }
 	    ent.set_fsa_status(tmp_inode->fsa_get_saved_status());
-	    if(tmp_inode->fsa_get_saved_status() == fsa_saved_status::full)
+	    if(tmp_inode->fsa_get_saved_status() == fsa_saved_status::full
+	       || tmp_inode->fsa_get_saved_status() == fsa_saved_status::partial)
 	    {
-		infinint tmp;
-
-		if(tmp_inode->fsa_get_offset(tmp))
-		    ent.set_archive_offset_for_FSA(tmp);
-		ent.set_storage_size_for_FSA(tmp_inode->fsa_get_size());
 		ent.set_fsa_scope(tmp_inode->fsa_get_families());
+
+		if(tmp_inode->fsa_get_saved_status() == fsa_saved_status::full)
+		{
+		    infinint tmp;
+
+		    if(tmp_inode->fsa_get_offset(tmp))
+			ent.set_archive_offset_for_FSA(tmp);
+		    ent.set_storage_size_for_FSA(tmp_inode->fsa_get_size());
+		}
 	    }
 	}
 

src/libdar/cat_inode.cpp

@@ -266,13 +266,13 @@ namespace libdar
 		switch(fsa_flag)
 		{
 		case INODE_FLAG_FSA_NONE:
-		    fsa_saved =  fsa_saved_status::none;
+		    fsa_saved = fsa_saved_status::none;
 		    break;
 		case INODE_FLAG_FSA_PART:
-		    fsa_saved =  fsa_saved_status::partial;
+		    fsa_saved = fsa_saved_status::partial;
 		    break;
 		case INODE_FLAG_FSA_FULL:
-		    fsa_saved =  fsa_saved_status::full;
+		    fsa_saved = fsa_saved_status::full;
 		    break;
 		default:
 		    throw Erange("cat_inode::cat_inode", gettext("badly structured inode: unknown inode flag for FSA"));
@@ -492,6 +492,9 @@ namespace libdar
 	    {
 		if(scope.size() > 0)
 		    throw Erange("cat_inode::compare", gettext("No Filesystem Specific Attribute to compare with"));
+
+		    // this is not perfect, the "other" could have no FSA due to the non empty scope excluding only
+		    // the valid FSAs. A more specific comparison would worth it...
 	    }
 	    break;
 	case fsa_saved_status::partial:
@@ -502,7 +505,13 @@ namespace libdar
                     throw Erange("cat_inode::compare", gettext("inode last change date (ctime) greater, FSA might be different"));
 	    }
 	    else
-		throw Erange("cat_inode::compare", gettext("Filesystem Specific Attribute are missing"));
+	    {
+		if(scope.size() > 0)
+		    throw Erange("cat_inode::compare", gettext("Filesystem Specific Attribute are missing"));
+
+		    // this is not perfect, the "other" could have no FSA due to the non empty scope excluding only
+		    // the valid FSAs. A more specific comparison would worth it...
+	    }
 	    break;
 	case fsa_saved_status::none:
 	    break; // nothing to check

src/libdar/fichier_libcurl.cpp

@@ -51,6 +51,7 @@ namespace libdar
 						  end_data_mode(false),
 						  sub_is_dying(false),
 						  sync_write_asked(false),
+						  weof(false),
 						  ehandle(handle),
 						  metadatamode(false),
 						  current_offset(0),
@@ -181,7 +182,7 @@ namespace libdar
 	    }
 	    catch(Egeneric & e)
 	    {
-		e.prepend_message("Error while reading file size on a remote repository");
+		e.prepend_message("Error while reading file size on a remote repository: ");
 		throw;
 	    }
 	}
@@ -526,6 +527,7 @@ namespace libdar
     void fichier_libcurl::initialize_subthread()
     {
 	sub_is_dying = false;
+	weof = false;
 	synchronize.wait(); // release calling thread as we, as child thread, do now exist
     }
 
@@ -805,10 +807,30 @@ namespace libdar
 	if(me == nullptr)
 	    throw SRC_BUG;
 
+	bool fetch_block = !me->weof || me->sync_write_asked;
+	    // with libcurl 7.74.0 and below, returning zero
+	    // bytes was properly addressed by libcurl as an EOF
+	    // and no more call to read_data_callback was "performed".
+	    // But with libcurl 7.88.1 read_data_callback() was called
+	    // again at least once after that, which lead fichier_libcurl
+	    // main thread hanging forever for curl_perform to end
+	    // because read_data_callback() was pending to receive
+	    // data from interthread pipe. To avoid that having the ending
+	    // of the subthread involving external software (libcurl) which
+	    // somehow changed without notice nor documentation change, we
+	    // add the weof boolean variable to avoid fetching from interthread
+	    // in that context.
+
 	do
 	{
 	    just_synced = false;
-	    me->interthread.fetch(ptr, ptr_size);
+	    if(fetch_block)
+		me->interthread.fetch(ptr, ptr_size);
+	    else // emulate an the fetching of a zero length block
+	    {
+		ptr_size = 0;
+		ptr = bufptr; // we will copy 0 bytes from bufptr to bufptr
+	    }
 
 		// note: if ptr_size is zero
 		// libcurl will assume EOF and stop
@@ -817,7 +839,8 @@ namespace libdar
 	    if(me->sync_write_asked && ptr_size == 0)
 	    {
 		me->sync_write_asked = false;
-		me->interthread.fetch_recycle(ptr);
+		if(fetch_block)
+		    me->interthread.fetch_recycle(ptr);
 		me->synchronize.wait();
 		just_synced = true;
 	    }
@@ -827,18 +850,23 @@ namespace libdar
 	if(ptr_size <= room)
 	{
 	    memcpy(bufptr, ptr, ptr_size);
-	    me->interthread.fetch_recycle(ptr);
+	    if(fetch_block)
+		me->interthread.fetch_recycle(ptr);
 	    ret = ptr_size;
 	}
 	else
 	{
 	    memcpy(bufptr, ptr, room);
 	    ptr_size -= room;
 	    memmove(ptr, ptr + room, ptr_size);
-	    me->interthread.fetch_push_back(ptr, ptr_size);
+	    if(fetch_block)
+		me->interthread.fetch_push_back(ptr, ptr_size);
 	    ret = room;
 	}
 
+	if(fetch_block && ptr_size == 0 && !(me->sync_write_asked))
+	    me->weof = true;
+
 	return ret;
     }
 

src/libdar/fichier_libcurl.hpp

@@ -155,6 +155,7 @@ namespace libdar
 	bool end_data_mode;               ///< true if subthread has been requested to end
 	bool sub_is_dying;                ///< is set by subthread when about to end
 	bool sync_write_asked;            ///< ask subthread to use 'synchronize' tell all data has been written
+	bool weof;                        ///< whether write EOF has been signaled to libcurl
 	std::shared_ptr<mycurl_easyhandle_node> ehandle; ///< easy handle (wrapped in C++ object) that we modify when necessary
 	bool metadatamode;                ///< wether we are acting on metadata rather than file's data
 	infinint current_offset;          ///< current offset we are reading / writing at

src/libdar/path.cpp

@@ -242,8 +242,8 @@ namespace libdar
 	if(me.size() >= you.size())
 	    if(strncmp(me.c_str(), you.c_str(), you.size()) == 0)
 		if(me.size() > you.size())
-		    return you.size() > 1 && me[you.size()] == '/'
-			|| you.size() == 1 && you[0] == '/';
+		    return (you.size() > 1 && me[you.size()] == '/')
+			|| (you.size() == 1 && you[0] == '/');
 		else  // thus, me.size() == you.size(), thus I'm a subdir of myself
 		    return true;
 	    else // path differs in the common length part, cannot be a subdir of "you"