Enable VST support on Linux

configure

@@ -637,6 +637,8 @@ ac_subst_vars='am__EXEEXT_FALSE
 am__EXEEXT_TRUE
 LTLIBOBJS
 LIBOBJS
+USE_VST_FALSE
+USE_VST_TRUE
 USE_QUICKTIME_FALSE
 USE_QUICKTIME_TRUE
 USE_AUDIO_UNITS_FALSE
@@ -975,6 +977,7 @@ with_widgetextra
 enable_audiounits
 enable_ladspa
 enable_quicktime
+enable_vst
 with_portmixer
 '
       ac_precious_vars='build_alias
@@ -1702,6 +1705,7 @@ Optional Features:
   --enable-ladspa         enable LADSPA plug-in support [default=yes]
   --enable-quicktime      enable QuickTime import support (Mac OS X only)
                           [default=auto]
+  --enable-vst            enable VST plug-in support [default=yes]
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
@@ -23624,6 +23628,14 @@ else
 fi
 
 
+# Check whether --enable-vst was given.
+if test "${enable_vst+set}" = set; then :
+  enableval=$enable_vst; use_vst=$enableval
+else
+  use_vst="yes"
+fi
+
+
 
 
 # Check whether --with-portmixer was given.
@@ -23637,6 +23649,7 @@ fi
 
 
 
+
 case "${host_os}" in
    darwin* | rhapsody*)
             CDEPEND="AudacityHeaders.h.gch"
@@ -24161,7 +24174,19 @@ $as_echo "#define USE_QUICKTIME 1" >>confdefs.h
 
 fi
 
+ if test "$use_vst" = yes; then
+  USE_VST_TRUE=
+  USE_VST_FALSE='#'
+else
+  USE_VST_TRUE='#'
+  USE_VST_FALSE=
+fi
 
+if [ "$use_vst" = "yes" ] ; then
+
+$as_echo "#define USE_VST 1" >>confdefs.h
+
+fi
 
 case "${host_os}" in
    cygwin*)
@@ -24483,6 +24508,10 @@ if test -z "${USE_QUICKTIME_TRUE}" && test -z "${USE_QUICKTIME_FALSE}"; then
   as_fn_error $? "conditional \"USE_QUICKTIME\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
 fi
+if test -z "${USE_VST_TRUE}" && test -z "${USE_VST_FALSE}"; then
+  as_fn_error $? "conditional \"USE_VST\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 
 : "${CONFIG_STATUS=./config.status}"
 ac_write_fail=0

configure.ac

@@ -550,6 +550,12 @@ AC_ARG_ENABLE(quicktime,
    use_quicktime=$enableval,
    use_quicktime="auto")
 
+AC_ARG_ENABLE(vst,
+   [AS_HELP_STRING([--enable-vst],
+      [enable VST plug-in support [default=yes]])],
+   use_vst=$enableval,
+   use_vst="yes")
+
 dnl In-tree libraries (ones we write, so only options are off and local)
 
 AC_ARG_WITH(portmixer,
@@ -558,6 +564,7 @@ AC_ARG_WITH(portmixer,
   use_portmixer=$withval,
   use_portmixer="yes")
 
+
 AC_CANONICAL_HOST
 
 dnl OS-specific configuration
@@ -680,7 +687,11 @@ if [[ "$use_quicktime" = "yes" ]] ; then
              [Define if QuickTime importing is enabled (Mac OS X only)])
 fi
 
-
+AM_CONDITIONAL([USE_VST], [test "$use_vst" = yes])
+if [[ "$use_vst" = "yes" ]] ; then
+   AC_DEFINE(USE_VST, 1,
+             [Define if VST plug-in support is enabled])
+fi
 
 case "${host_os}" in
    cygwin*)

src/Makefile.am

@@ -58,6 +58,24 @@ audacity_CPPFLAGS = \
 	$(WX_CXXFLAGS) \
 	$(NULL)
 
+# Until we upgrade to a newer version of wxWidgets...will get rid of hundreds of these:
+#
+#  /usr/local/include/wx-2.8/wx/buffer.h: In member function 'void wxMemoryBuffer::AppendByte(char)':
+#  /usr/local/include/wx-2.8/wx/debug.h:194:43: warning: typedef 'wxDummyCheckInt' locally defined but not used [-Wunused-local-typedefs]
+#       #define wxFORCE_SEMICOLON typedef int wxDummyCheckInt
+#                                           ^
+#  /usr/local/include/wx-2.8/wx/debug.h:224:9: note: in expansion of macro 'wxFORCE_SEMICOLON'
+#           wxFORCE_SEMICOLON /* just to force a semicolon */
+#           ^
+#  /usr/local/include/wx-2.8/wx/debug.h:233:38: note: in expansion of macro 'wxCHECK2_MSG'
+#   #define wxCHECK_RET(cond, msg)       wxCHECK2_MSG(cond, return, msg)
+#                                        ^
+#  /usr/local/include/wx-2.8/wx/buffer.h:289:9: note: in expansion of macro 'wxCHECK_RET'
+#           wxCHECK_RET( m_bufdata->m_data, wxT("invalid wxMemoryBuffer") );
+#           ^
+audacity_CXXFLAGS = \
+   -Wno-unused-local-typedefs
+
 audacity_LDFLAGS = -rdynamic
 audacity_LDADD = \
 	$(EXPAT_LIBS) \
@@ -358,9 +376,6 @@ audacity_SOURCES = \
 	effects/TwoPassSimpleMono.h \
 	effects/Wahwah.cpp \
 	effects/Wahwah.h \
-	effects/VST/aeffectx.h \
-	effects/VST/VSTEffect.cpp \
-	effects/VST/VSTEffect.h \
 	export/Export.cpp \
 	export/Export.h \
 	export/ExportCL.cpp \
@@ -671,6 +686,16 @@ audacity_SOURCES += \
 	$(NULL)
 endif
 
+if USE_VST
+audacity_CPPFLAGS += $(VST_CFLAGS)
+audacity_LDADD += $(VST_LIBS)
+audacity_SOURCES += \
+	effects/VST/aeffectx.h \
+	effects/VST/VSTEffect.cpp \
+	effects/VST/VSTEffect.h \
+	$(NULL)
+endif
+
 # TODO: Check *.cpp and *.h files if they are needed.
 EXTRA_DIST = audacity.desktop.in xml/audacityproject.dtd \
 	AudacityHeaders.cpp \

src/configtemplate.h

@@ -182,6 +182,9 @@
 /* Define if Vamp analysis plugin support should be enabled */
 #undef USE_VAMP
 
+/* Define if VST plugin support should be enabled */
+#undef USE_VST
+
 /* Version number of package */
 #undef VERSION
 

src/effects/VST/VSTEffect.cpp

@@ -943,6 +943,7 @@ private:
 #elif defined(__WXMSW__)
    wxSizerItem *mContainer;
 #else
+   wxSizerItem *mContainer;
 #endif
 
    wxComboBox *mProgram;
@@ -1348,6 +1349,15 @@ VSTEffectDialog::VSTEffectDialog(wxWindow *parent,
    mGui = (gPrefs->Read(wxT("/VST/GUI"), (long) true) != 0) &&
           mAEffect->flags & effFlagsHasEditor;
 
+#if defined(__WXGTK__)
+   // Let the user know that a GUI interface is not supported in wxGTK
+   if (mGui) {
+      wxMessageBox(_("VST GUI interfaces are currently unsupported on Linux"),
+                   _("VST Effect"));
+      mGui = false;
+   }
+#endif
+
    // Build the appropriate dialog type
    if (mGui) {
       BuildFancy();
@@ -1456,6 +1466,7 @@ void VSTEffectDialog::BuildFancy()
    mEffect->callDispatcher(effEditOpen, 0, 0, w->GetHWND(), 0.0);
 
 #else
+
 #endif
 
    // Get the final bounds of the effect GUI
@@ -1479,6 +1490,7 @@ void VSTEffectDialog::BuildFancy()
    mContainer->SetMinSize(rect->right - rect->left, rect->bottom - rect->top);
 
 #else
+
 #endif
 
    vs->Add(hs, 0, wxCENTER);
@@ -2066,8 +2078,11 @@ void VSTEffectDialog::OnPreview(wxCommandEvent & WXUNUSED(event))
 
 void VSTEffectDialog::OnOk(wxCommandEvent & WXUNUSED(event))
 {
+// In wxGTK, Show(false) calls EndModal, which produces an assertion in debug builds
+#if !defined(__WXGTK__)
    // Hide the dialog before closing the effect to prevent a brief empty dialog
    Show(false);
+#endif
 
    if (mGui) {
       mEffect->callDispatcher(effEditClose, 0, 0, NULL, 0.0);
@@ -2078,8 +2093,11 @@ void VSTEffectDialog::OnOk(wxCommandEvent & WXUNUSED(event))
 
 void VSTEffectDialog::OnCancel(wxCommandEvent & WXUNUSED(event))
 {
+// In wxGTK, Show(false) calls EndModal, which produces an assertion in debug builds
+#if !defined(__WXGTK__)
    // Hide the dialog before closing the effect to prevent a brief empty dialog
    Show(false);
+#endif
 
    if (mGui) {
       mEffect->callDispatcher(effEditClose, 0, 0, NULL, 0.0);
@@ -3116,6 +3134,7 @@ bool VSTEffect::ProcessStereo(int count,
 void VSTEffect::End()
 {
 }
+#include <dlfcn.h>
 
 bool VSTEffect::Load()
 {