Fix VST sub-window creation glitches in project loading

Fixes bugs where during project loading (observed with VST effects), empty
widgets and sub-windows would be left floating around. These were caused by
inconsistencies between the way VST UIs were created when loading a project
and when adding an effect in an existing project. In some situations, this
caused createUI to be called twice, leaving over multiple empty widgets.

This commit refactors some code in order to avoid creating unnecessary sub-
windows, which aren't needed with VST effects, but were still created,
usually being invisible. All sub-window related code was moved out of
VstPlugin into vestige.cpp, which is the only place where sub-window VSTs
are actually used. A new sub-class of VstPlugin, VstInstrumentPlugin, now
handles VST sub-windows and is used by vestigeInstrument.

"guivisible" attribute loading was moved out of VstPlugin as well and is
now done in VstEffectControls' and vestigeInstrument's loadSettings method
respectively. This causes some minor code duplication unfortunately.

Closes #4110

plugins/VstEffect/VstEffectControlDialog.cpp

@@ -45,6 +45,7 @@
 VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 	EffectControlDialog( _ctl ),
 	m_pluginWidget( NULL ),
+
 	m_plugin( NULL ),
 	tbLabel( NULL )
 {
@@ -62,39 +63,31 @@ VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 		embed_vst = m_plugin->embedMethod() != "none";
 
 		if (embed_vst) {
-			m_plugin->createUI( nullptr, true );
-			m_pluginWidget = m_plugin->pluginWidget( false );
-
-#ifdef LMMS_BUILD_WIN32
-			if( !m_pluginWidget )
-			{
-				m_pluginWidget = m_plugin->pluginWidget( false );
+			if (! m_plugin->pluginWidget()) {
+				m_plugin->createUI(nullptr);
 			}
-#endif
-
+			m_pluginWidget = m_plugin->pluginWidget();
 		}
 	}
 
 	if ( m_plugin && (!embed_vst || m_pluginWidget) )
 	{
 		setWindowTitle( m_plugin->name() );
 
-		QPushButton * btn = new QPushButton( tr( "Show/hide" ) );
+		QPushButton * btn = new QPushButton( tr( "Show/hide" ));
 
 		if (embed_vst) {
 			btn->setCheckable( true );
 			btn->setChecked( true );
-			connect( btn, SIGNAL( toggled( bool ) ),
-						SLOT( togglePluginUI( bool ) ) );
-		} else {
-			connect( btn, SIGNAL( clicked( bool ) ),
-						SLOT( togglePluginUI( bool ) ) );
 		}
+		connect( btn, SIGNAL( toggled( bool ) ),
+					SLOT( togglePluginUI( bool ) ) );
 
 		btn->setMinimumWidth( 78 );
 		btn->setMaximumWidth( 78 );
 		btn->setMinimumHeight( 24 );
 		btn->setMaximumHeight( 24 );
+		m_togglePluginButton = btn;
 
 		m_managePluginButton = new PixmapButton( this, "" );
 		m_managePluginButton->setCheckable( false );
@@ -295,6 +288,14 @@ void VstEffectControlDialog::togglePluginUI( bool checked )
 		return;
 	}
 
-	m_plugin->toggleUI();
+	if ( m_togglePluginButton->isChecked() != checked ) {
+		m_togglePluginButton->setChecked( checked );
+	}
+
+	if ( checked ) {
+		m_plugin->showUI();
+	} else {
+		m_plugin->hideUI();
+	}
 }
 

plugins/VstEffect/VstEffectControlDialog.h

@@ -54,6 +54,7 @@ class VstEffectControlDialog : public EffectControlDialog
 private:
 	QWidget * m_pluginWidget;
 
+	QPushButton * m_togglePluginButton;
 	PixmapButton * m_openPresetButton;
 	PixmapButton * m_rolLPresetButton;
 	PixmapButton * m_rolRPresetButton;
@@ -64,7 +65,7 @@ class VstEffectControlDialog : public EffectControlDialog
 
 	QLabel * tbLabel;
 
-private slots:
+public slots:
 	void togglePluginUI( bool checked );
 } ;
 

plugins/VstEffect/VstEffectControls.cpp

@@ -40,7 +40,8 @@ VstEffectControls::VstEffectControls( VstEffect * _eff ) :
 	m_subWindow( NULL ),
 	knobFModel( NULL ),
 	ctrHandle( NULL ),
-	lastPosInMenu (0)
+	lastPosInMenu (0),
+	m_vstGuiVisible ( true )
 //	m_presetLabel ( NULL )
 {
 }
@@ -64,6 +65,8 @@ void VstEffectControls::loadSettings( const QDomElement & _this )
 	m_effect->m_pluginMutex.lock();
 	if( m_effect->m_plugin != NULL )
 	{
+		m_vstGuiVisible = _this.attribute( "guivisible" ).toInt();
+
 		m_effect->m_plugin->loadSettings( _this );
 
 		const QMap<QString, QString> & dump = m_effect->m_plugin->parameterDump();
@@ -144,6 +147,15 @@ int VstEffectControls::controlCount()
 
 
 
+EffectControlDialog *VstEffectControls::createView()
+{
+	auto dialog = new VstEffectControlDialog( this );
+	dialog->togglePluginUI( m_vstGuiVisible );
+	return dialog;
+}
+
+
+
 
 void VstEffectControls::managePlugin( void )
 {

plugins/VstEffect/VstEffectControls.h

@@ -59,10 +59,7 @@ class VstEffectControls : public EffectControls
 
 	virtual int controlCount();
 
-	virtual EffectControlDialog * createView()
-	{
-		return new VstEffectControlDialog( this );
-	}
+	virtual EffectControlDialog * createView();
 
 
 protected slots:
@@ -96,6 +93,7 @@ protected slots:
 	friend class VstEffectControlDialog;
 	friend class manageVSTEffectView;
 
+	bool m_vstGuiVisible;
 } ;
 
 

plugins/vestige/vestige.cpp

@@ -73,6 +73,56 @@ Plugin::Descriptor PLUGIN_EXPORT vestige_plugin_descriptor =
 }
 
 
+class vstSubWin : public QMdiSubWindow
+{
+public:
+	vstSubWin( QWidget * _parent ) :
+		QMdiSubWindow( _parent )
+	{
+		setAttribute( Qt::WA_DeleteOnClose, false );
+		setWindowFlags( Qt::WindowCloseButtonHint );
+	}
+
+	virtual ~vstSubWin()
+	{
+	}
+
+	virtual void closeEvent( QCloseEvent * e )
+	{
+		// ignore close-events - for some reason otherwise the VST GUI
+		// remains hidden when re-opening
+		hide();
+		e->ignore();
+	}
+};
+
+
+class VstInstrumentPlugin : public VstPlugin
+{
+public:
+	using VstPlugin::VstPlugin;
+
+	void createUI( QWidget *parent ) override
+	{
+		Q_UNUSED(parent);
+		VstPlugin::createUI( nullptr );
+		if ( embedMethod() != "none" ) {
+			m_pluginSubWindow.reset(new vstSubWin( gui->mainWindow()->workspace() ));
+			m_pluginSubWindow->setWidget(pluginWidget());
+		}
+	}
+
+	/// Overwrite editor() to return the sub window instead of the embed widget
+	/// itself. This makes hideEditor() and toggleEditor() hide the sub window.
+	QWidget* editor() override
+	{
+		return m_pluginSubWindow.get();
+	}
+private:
+	unique_ptr<QMdiSubWindow> m_pluginSubWindow;
+};
+
+
 QPixmap * VestigeInstrumentView::s_artwork = NULL;
 QPixmap * manageVestigeInstrumentView::s_artwork = NULL;
 
@@ -127,6 +177,13 @@ void vestigeInstrument::loadSettings( const QDomElement & _this )
 	{
 		m_plugin->loadSettings( _this );
 
+		m_plugin->createUI( nullptr );
+		if ( _this.attribute( "guivisible" ).toInt() ) {
+			m_plugin->showUI();
+		} else {
+			m_plugin->hideUI();
+		}
+
 		const QMap<QString, QString> & dump = m_plugin->parameterDump();
 		paramCount = dump.size();
 		char paramStr[35];
@@ -267,7 +324,7 @@ void vestigeInstrument::loadFile( const QString & _file )
 	}
 
 	m_pluginMutex.lock();
-	m_plugin = new VstPlugin( m_pluginDLL );
+	m_plugin = new VstInstrumentPlugin( m_pluginDLL );
 	if( m_plugin->failed() )
 	{
 		m_pluginMutex.unlock();
@@ -278,6 +335,7 @@ void vestigeInstrument::loadFile( const QString & _file )
 		return;
 	}
 
+	m_plugin->createUI(nullptr);
 	m_plugin->showUI();
 
 	if( set_ch_name )