Merge pull request #2289 from AllenInstitute/bugfix/2289-fix-querying…

…-labnotebook-unknown-mode-nan-sweep

Labnotebook querying: Fix querying the NaN sweep with unknown mode

Packages/MIES/MIES_Constants.ipf

@@ -38,7 +38,7 @@ Constant SWEEP_EPOCH_VERSION = 9
 /// - New/Changed layers of entries
 ///
 ///@{
-Constant LABNOTEBOOK_VERSION = 76
+Constant LABNOTEBOOK_VERSION = 77
 Constant RESULTS_VERSION     = 3
 ///@}
 

Packages/MIES/MIES_MiesUtilities_Logbook.ipf

@@ -1368,18 +1368,30 @@ threadsafe static Function FindRange(wv, col, val, entrySourceType, first, last)
 	variable col, val, entrySourceType
 	variable &first, &last
 
-	variable numRows, i, j, sourceTypeCol, firstRow, lastRow, isNumeric, index
+	variable numRows, i, j, sourceTypeCol, firstRow, lastRow, isNumeric, index, startRow, endRow
 
 	first     = NaN
 	last      = NaN
 	isNumeric = IsNumericWave(wv)
 
+	startRow = 0
+	endRow   = GetNumberFromWaveNote(wv, NOTE_INDEX) - 1
+
+	if(IsNaN(endRow))
+		endRow = DimSize(wv, ROWS) - 1
+	endif
+
+	if(endRow < 0)
+		// empty labnotebook
+		return NaN
+	endif
+
 	// still correct without startLayer/endLayer coordinates
 	// as we always have sweepNumber/etc. in the first layer
 	if(IsNaN(val) && isNumeric)
-		WAVE/Z indizesSetting = FindIndizes(wv, col = col, prop = PROP_EMPTY)
+		WAVE/Z indizesSetting = FindIndizes(wv, col = col, prop = PROP_EMPTY, startRow = startRow, endRow = endRow)
 	else
-		WAVE/Z indizesSetting = FindIndizes(wv, col = col, var = val)
+		WAVE/Z indizesSetting = FindIndizes(wv, col = col, var = val, startRow = startRow, endRow = endRow)
 	endif
 
 	if(!WaveExists(indizesSetting))

Packages/MIES/MIES_WaveDataFolderGetters.ipf

@@ -1396,7 +1396,7 @@ End
 static Function UpgradeLabNotebook(device)
 	string device
 
-	variable numCols, i, col, numEntries, sourceCol
+	variable numCols, i, col, numEntries, sourceCol, timeStampColumn, nextFreeRow
 	string list, key
 
 	// we only have to check the new place and name as we are called
@@ -1633,8 +1633,40 @@ static Function UpgradeLabNotebook(device)
 	endif
 	// END fix missing column dimension labels in keyWaves
 
-	// we don't remove the wavenote entry as we might need to adapt the reading code
-	// in the future to handle labnotebooks with sweep rollback specially.
+	// we don't remove the wavenote entry of sweep rollback as we might need to adapt the reading code
+	// in the future to handle labnotebooks with that specially
+
+	// BEGIN add note index
+	// Timestamp was always present in column 1, see879683fd1 (-InitiateMies: Updated comments
+	// -WaveDataFolderGetter: Updated comments, changed order to rows, columns, layers, 2014-09-12), but spelled
+	// differently prior to ec6c1ac6b (Labnotebook: Add UTC timestamps, 2015-09-18)
+	timeStampColumn = 1
+
+	if(WaveVersionIsSmaller(numericalKeys, 77))
+		nextFreeRow = GetNumberFromWaveNote(numericalValues, NOTE_INDEX)
+
+		if(IsNaN(nextFreeRow))
+			FindValue/FNAN/RMD=[][timeStampColumn][0]/R numericalValues
+			if(!(V_row >= 0))
+				// labnotebook is completely full
+				V_row = DimSize(numericalValues, ROWS)
+			endif
+			SetNumberInWaveNote(numericalValues, NOTE_INDEX, V_row)
+		endif
+	endif
+
+	if(WaveVersionIsSmaller(textualKeys, 77))
+		nextFreeRow = GetNumberFromWaveNote(textualValues, NOTE_INDEX)
+
+		if(IsNaN(nextFreeRow))
+			FindValue/FNAN/RMD=[][timeStampColumn][0]/R textualValues
+			if(!(V_row >= 0))
+				V_row = DimSize(textualValues, ROWS)
+			endif
+			SetNumberInWaveNote(textualValues, NOTE_INDEX, V_row)
+		endif
+	endif
+	// END add note index
 End
 
 static Function/S FixInvalidLabnotebookKey(string name)

Packages/tests/Basic/UTF_Labnotebook.ipf

@@ -12,17 +12,25 @@ End
 
 static Function/WAVE PrepareLBNNumericalValues(WAVE numericalValuesSrc)
 
-	variable numCols
+	variable numCols, nextFreeRow, numRows
 
 	WAVE/T numericalKeysTemplate = GetLBNumericalKeys("dummyDevice")
 	numCols = DimSize(numericalValuesSrc, COLS)
 	Redimension/N=(-1, numCols, -1, -1) numericalKeysTemplate
 	numericalKeysTemplate[0][] = GetDimLabel(numericalValuesSrc, COLS, q)
 
-	Duplicate/O numericalValuesSrc, $LBN_NUMERICAL_VALUES_NAME
+	Duplicate/O numericalValuesSrc, $LBN_NUMERICAL_VALUES_NAME/WAVE=numericalValues
 	Duplicate/O numericalKeysTemplate, $LBN_NUMERICAL_KEYS_NAME
 
-	return $LBN_NUMERICAL_VALUES_NAME
+	// fixup NOTE_INDEX which is wrong due to optimizing the wave sizes for the tests
+	nextFreeRow = GetNumberFromWaveNote(numericalValues, NOTE_INDEX)
+	numRows     = DimSize(numericalValues, ROWS)
+
+	if(nextFreeRow > numRows)
+		SetNumberInWaveNote(numericalValues, NOTE_INDEX, numRows)
+	endif
+
+	return numericalValues
 End
 
 static Function/WAVE PrepareLBNTextualValues(WAVE textualValuesSrc)
@@ -208,6 +216,10 @@ Function GetLastSettingFindsNaNSweep()
 
 	Make/D/FREE settingsRef = {10.0010900497437, 10.001935005188, NaN, NaN, NaN, NaN, NaN, NaN, NaN}
 	CHECK_EQUAL_WAVES(settings, settingsRef, mode = WAVE_DATA, tol = 1e-13)
+
+	// and also with unknown mode
+	WAVE/Z settings = GetLastSetting(numericalValues, NaN, "TP Steady State Resistance", UNKNOWN_MODE)
+	CHECK_EQUAL_WAVES(settings, settingsRef, mode = WAVE_DATA, tol = 1e-13)
 End
 
 static Function GetLastSettingFindsWithinNonConsecutiveSweepOrder()
@@ -1339,3 +1351,93 @@ Function GetUniqueSettingsWorks()
 	CHECK_WAVE(resultsTxt, TEXT_WAVE)
 	CHECK_EQUAL_TEXTWAVES(resultsTxt, {"131415", "192021", "161718", "222324", "252627"})
 End
+
+Function LabnotebookUpgradeMissingNoteIndexNumerical()
+
+	variable idx, idxRedone
+	string device, key, keyTxt
+
+	device = "ITC16USB_0_DEV"
+	[key, keyTxt] = PrepareLBN_IGNORE(device)
+
+	WAVE/Z   numericalValues = GetLBNumericalValues(device)
+	WAVE/T/Z numericalKeys   = GetLBNumericalKeys(device)
+
+	idx = GetNumberFromWaveNote(numericalValues, NOTE_INDEX)
+	CHECK_GT_VAR(idx, 0)
+
+	Note/K numericalKeys
+
+	MIES_WAVEGETTERS#UpgradeLabNotebook(device)
+
+	idxRedone = GetNumberFromWaveNote(numericalValues, NOTE_INDEX)
+	CHECK_EQUAL_VAR(idxRedone, idx)
+
+	Note/K numericalKeys
+	Note/K numericalValues
+
+	numericalValues[][][] = NaN
+	Redimension/N=(0, -1, -1) numericalValues
+
+	MIES_WAVEGETTERS#UpgradeLabNotebook(device)
+
+	idxRedone = GetNumberFromWaveNote(numericalValues, NOTE_INDEX)
+	CHECK_EQUAL_VAR(idxRedone, 0)
+End
+
+Function LabnotebookUpgradeMissingNoteIndexTextual()
+
+	variable idx, idxRedone
+	string device, key, keyTxt
+
+	device = "ITC16USB_0_DEV"
+	[key, keyTxt] = PrepareLBN_IGNORE(device)
+
+	WAVE/T/Z textualValues = GetLBTextualValues(device)
+	WAVE/T/Z textualKeys   = GetLBTextualKeys(device)
+
+	idx = GetNumberFromWaveNote(textualValues, NOTE_INDEX)
+	CHECK_GT_VAR(idx, 0)
+
+	Note/K textualKeys
+
+	MIES_WAVEGETTERS#UpgradeLabNotebook(device)
+
+	idxRedone = GetNumberFromWaveNote(textualValues, NOTE_INDEX)
+	CHECK_EQUAL_VAR(idxRedone, idx)
+
+	Note/K textualValues
+	Note/K textualKeys
+
+	textualValues[][][] = ""
+	Redimension/N=(0, -1, -1) textualValues
+
+	MIES_WAVEGETTERS#UpgradeLabNotebook(device)
+
+	idxRedone = GetNumberFromWaveNote(textualValues, NOTE_INDEX)
+	CHECK_EQUAL_VAR(idxRedone, 0)
+End
+
+Function EmptyLabnotebookWorks()
+
+	string device
+
+	device = "ITC16USB_0_DEV"
+	WAVE/Z numericalValues = GetLBNumericalValues(device)
+	CHECK_WAVE(numericalValues, NUMERIC_WAVE)
+
+	WAVE/Z entries = GetLastSetting(numericalValues, NaN, "Sweep Number", UNKNOWN_MODE)
+	CHECK_WAVE(entries, NULL_WAVE)
+
+	WAVE/Z entries = GetLastSetting(numericalValues, 0, "Sweep Number", UNKNOWN_MODE)
+	CHECK_WAVE(entries, NULL_WAVE)
+
+	WAVE/Z textualValues = GetLBTextualValues(device)
+	CHECK_WAVE(textualValues, TEXT_WAVE)
+
+	WAVE/Z entries = GetLastSetting(textualValues, NaN, "Sweep Number", UNKNOWN_MODE)
+	CHECK_WAVE(entries, NULL_WAVE)
+
+	WAVE/Z entries = GetLastSetting(textualValues, 0, "Sweep Number", UNKNOWN_MODE)
+	CHECK_WAVE(entries, NULL_WAVE)
+End