diff --git a/active_directory/tests/test_active_directory.py b/active_directory/tests/test_active_directory.py
index 0ac72e1b6d388..2b05be6bb45e0 100644
--- a/active_directory/tests/test_active_directory.py
+++ b/active_directory/tests/test_active_directory.py
@@ -14,10 +14,12 @@
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 MINIMAL_INSTANCE = {
+    'cache_counter_instances': False,
     'host': '.',
 }
 
 INSTANCE_WITH_TAGS = {
+    'cache_counter_instances': False,
     'host': '.',
     'tags': ['tag1', 'another:tag']
 }
diff --git a/active_directory/tests/test_bench.py b/active_directory/tests/test_bench.py
new file mode 100644
index 0000000000000..5845640602bd9
--- /dev/null
+++ b/active_directory/tests/test_bench.py
@@ -0,0 +1,37 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+from datadog_checks.active_directory import ActiveDirectoryCheck
+from datadog_test_libs.win.pdh_mocks import pdh_mocks_fixture, initialize_pdh_tests  # noqa: F401
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': True,
+        'host': '.',
+    }
+    check = ActiveDirectoryCheck('active_directory', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_no_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': False,
+        'host': '.',
+    }
+    check = ActiveDirectoryCheck('active_directory', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
diff --git a/active_directory/tox.ini b/active_directory/tox.ini
index f6d3fcc07ac0c..13a4496dd18a0 100644
--- a/active_directory/tox.ini
+++ b/active_directory/tox.ini
@@ -4,12 +4,10 @@ basepython = py27
 envlist =
     active_directory
     flake8
+    bench
 
 [testenv]
 usedevelop = true
-platform = win32|linux2|darwin
-
-[testenv:active_directory]
 platform = win32
 deps =
     -e../datadog_checks_base[deps]
@@ -17,13 +15,18 @@ deps =
     -rrequirements-dev.txt
 commands =
     pip install -r requirements.in
-    pytest -v
+    pytest -v --benchmark-skip
 
 [testenv:flake8]
 skip_install = true
 deps = flake8
 commands = flake8 .
 
+[testenv:bench]
+commands =
+    pip install -r requirements.in
+    pytest --benchmark-only --benchmark-cprofile=tottime
+
 [flake8]
 exclude = .eggs,.tox,build
 max-line-length = 120
diff --git a/appveyor.yml b/appveyor.yml
index 524dcff9d80f6..3e8c498d6d035 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -46,7 +46,8 @@ install:
 
 test_script:
   # Only test any of these that have changed for pull requests
-  - ddev test%CHANGED_ONLY_FLAG% -c datadog_checks_base active_directory aspdotnet disk dotnetclr exchange_server iis pdh_check sqlserver win32_event_log windows_service wmi_check
+  - ddev test datadog_checks_base active_directory aspdotnet dotnetclr exchange_server iis pdh_check
+  - ddev test -b active_directory aspdotnet dotnetclr exchange_server iis pdh_check
 
 # Uncomment the following to enable RDP connection into the builder and debug a build
 # on_finish:
diff --git a/aspdotnet/tests/test_aspdotnet.py b/aspdotnet/tests/test_aspdotnet.py
index d993196b6b9f7..fc3a8f64ee7ac 100644
--- a/aspdotnet/tests/test_aspdotnet.py
+++ b/aspdotnet/tests/test_aspdotnet.py
@@ -13,10 +13,12 @@
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 MINIMAL_INSTANCE = {
+    'cache_counter_instances': False,
     'host': '.',
 }
 
 INSTANCE_WITH_TAGS = {
+    'cache_counter_instances': False,
     'host': '.',
     'tags': ['tag1', 'another:tag']
 }
diff --git a/aspdotnet/tests/test_bench.py b/aspdotnet/tests/test_bench.py
new file mode 100644
index 0000000000000..63e018b47397d
--- /dev/null
+++ b/aspdotnet/tests/test_bench.py
@@ -0,0 +1,37 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+from datadog_checks.aspdotnet import AspdotnetCheck
+from datadog_test_libs.win.pdh_mocks import pdh_mocks_fixture, initialize_pdh_tests  # noqa: F401
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': True,
+        'host': '.',
+    }
+    check = AspdotnetCheck('aspdotnet', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_no_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': False,
+        'host': '.',
+    }
+    check = AspdotnetCheck('aspdotnet', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
diff --git a/aspdotnet/tox.ini b/aspdotnet/tox.ini
index 56269f54e0287..14b9c799175e9 100644
--- a/aspdotnet/tox.ini
+++ b/aspdotnet/tox.ini
@@ -4,12 +4,10 @@ basepython = py27
 envlist =
     aspdotnet
     flake8
+    bench
 
 [testenv]
 usedevelop = true
-platform = win32|linux2|darwin
-
-[testenv:aspdotnet]
 platform = win32
 deps =
     -e../datadog_checks_base[deps]
@@ -17,13 +15,18 @@ deps =
     -rrequirements-dev.txt
 commands =
     pip install -r requirements.in
-    pytest -v
+    pytest -v --benchmark-skip
 
 [testenv:flake8]
 skip_install = true
 deps = flake8
 commands = flake8 .
 
+[testenv:bench]
+commands =
+    pip install -r requirements.in
+    pytest --benchmark-only --benchmark-cprofile=tottime
+
 [flake8]
 exclude = .eggs,.tox,build
 max-line-length = 120
diff --git a/datadog_checks_base/datadog_checks/base/checks/win/winpdh.py b/datadog_checks_base/datadog_checks/base/checks/win/winpdh.py
index 59d42d38e1c7e..d6939eae0304c 100644
--- a/datadog_checks_base/datadog_checks/base/checks/win/winpdh.py
+++ b/datadog_checks_base/datadog_checks/base/checks/win/winpdh.py
@@ -20,7 +20,17 @@ class WinPDHCounter(object):
     _use_en_counter_names = False
 
     def __init__(self, class_name, counter_name, log, instance_name=None, machine_name=None, precision=None):
+        self.counterdict = {}
         self.logger = log
+        self._counter_name = counter_name
+        self._instance_name = instance_name
+        self._machine_name = machine_name
+        self._is_single_instance = False
+
+        if precision is None:
+            self._precision = win32pdh.PDH_FMT_DOUBLE
+        else:
+            self._precision = precision
 
         class_name_index_list = []
         try:
@@ -46,65 +56,9 @@ def __init__(self, class_name, counter_name, log, instance_name=None, machine_na
                     )
                 self._class_name = win32pdh.LookupPerfNameByIndex(None, int(class_name_index_list[0]))
 
-        self._is_single_instance = False
         self.hq = win32pdh.OpenQuery()
+        self.collect_counters()
 
-        self.counterdict = {}
-        if precision is None:
-            self._precision = win32pdh.PDH_FMT_DOUBLE
-        else:
-            self._precision = precision
-        counters, instances = win32pdh.EnumObjectItems(
-            None, machine_name, self._class_name, win32pdh.PERF_DETAIL_WIZARD
-        )
-        if instance_name is None and len(instances) > 0:
-            for inst in instances:
-                path = self._make_counter_path(machine_name, counter_name, inst, counters)
-                if not path:
-                    continue
-                try:
-                    self.counterdict[inst] = win32pdh.AddCounter(self.hq, path)
-                except:  # noqa: E722
-                    self.logger.fatal("Failed to create counter.  No instances of %s\\%s" % (
-                        self._class_name, self._counter_name))
-                try:
-                    self.logger.debug("Path: %s\n" % text_type(path))
-                except:  # noqa: E722
-                    # some unicode characters are not translatable here.  Don't fail just
-                    # because we couldn't log
-                    self.logger.debug("Failed to log path")
-        else:
-            if instance_name is not None:
-                # check to see that it's valid
-                if len(instances) <= 0:
-                    self.logger.error(
-                        "%s doesn't seem to be a multi-instance counter, but asked for specific instance %s" % (
-                            class_name, instance_name
-                        )
-                    )
-                    raise AttributeError("%s is not a multi-instance counter" % class_name)
-                if instance_name not in instances:
-                    self.logger.error("%s is not a counter instance in %s" % (
-                        instance_name, class_name
-                    ))
-                    raise AttributeError("%s is not an instance of %s" % (instance_name, class_name))
-            path = self._make_counter_path(machine_name, counter_name, instance_name, counters)
-            if not path:
-                self.logger.warning("Empty path returned")
-            else:
-                try:
-                    self.logger.debug("Path: %s\n" % text_type(path))
-                except:  # noqa: E722
-                    # some unicode characters are not translatable here.  Don't fail just
-                    # because we couldn't log
-                    self.logger.debug("Failed to log path")
-                try:
-                    self.counterdict[SINGLE_INSTANCE_KEY] = win32pdh.AddCounter(self.hq, path)
-                except:  # noqa: E722
-                    self.logger.fatal("Failed to create counter.  No instances of %s\\%s" % (
-                        self._class_name, counter_name))
-                    raise
-                self._is_single_instance = True
         if len(self.counterdict) == 0:
             raise AttributeError("No valid counters to report")
 
@@ -231,7 +185,6 @@ def _make_counter_path(self, machine_name, counter_name, instance_name, counters
             # see if we can create a counter
             try:
                 path = win32pdh.MakeCounterPath((machine_name, self._class_name, instance_name, None, 0, c))
-                self.logger.debug("Successfully created path %s" % index)
                 break
             except:  # noqa: E722
                 try:
@@ -239,3 +192,60 @@ def _make_counter_path(self, machine_name, counter_name, instance_name, counters
                 except:  # noqa: E722
                     self.logger.info("Unable to make path with counter index %s, trying next available" % index)
         return path
+
+    def collect_counters(self):
+        counters, instances = win32pdh.EnumObjectItems(
+            None, self._machine_name, self._class_name, win32pdh.PERF_DETAIL_WIZARD
+        )
+        if self._instance_name is None and len(instances) > 0:
+            all_instances = set()
+            for inst in instances:
+                path = self._make_counter_path(self._machine_name, self._counter_name, inst, counters)
+                if not path:
+                    continue
+                all_instances.add(inst)
+
+                try:
+                    if inst not in self.counterdict:
+                        self.logger.debug('Adding instance `{}`'.format(inst))
+                        self.counterdict[inst] = win32pdh.AddCounter(self.hq, path)
+                except:  # noqa: E722
+                    self.logger.fatal("Failed to create counter.  No instances of %s\\%s" % (
+                        self._class_name, self._counter_name))
+
+            expired_instances = set(self.counterdict) - all_instances
+            for inst in expired_instances:
+                self.logger.debug('Removing expired instance `{}`'.format(inst))
+                del self.counterdict[inst]
+        else:
+            if self._instance_name is not None:
+                # check to see that it's valid
+                if len(instances) <= 0:
+                    self.logger.error(
+                        "%s doesn't seem to be a multi-instance counter, but asked for specific instance %s" % (
+                            self._class_name, self._instance_name
+                        )
+                    )
+                    raise AttributeError("%s is not a multi-instance counter" % self._class_name)
+                if self._instance_name not in instances:
+                    self.logger.error("%s is not a counter instance in %s" % (
+                        self._instance_name, self._class_name
+                    ))
+                    raise AttributeError("%s is not an instance of %s" % (self._instance_name, self._class_name))
+
+            path = self._make_counter_path(self._machine_name, self._counter_name, self._instance_name, counters)
+            if not path:
+                self.logger.warning("Empty path returned")
+            elif win32pdh.ValidatePath(path) != 0:
+                # Multi-instance counter with no instances presently
+                pass
+            else:
+                try:
+                    if SINGLE_INSTANCE_KEY not in self.counterdict:
+                        self.logger.debug('Adding single instance for path `{}`'.format(path))
+                        self.counterdict[SINGLE_INSTANCE_KEY] = win32pdh.AddCounter(self.hq, path)
+                except:  # noqa: E722
+                    self.logger.fatal("Failed to create counter.  No instances of %s\\%s" % (
+                        self._class_name, self._counter_name))
+                    raise
+                self._is_single_instance = True
diff --git a/datadog_checks_base/datadog_checks/base/checks/win/winpdh_base.py b/datadog_checks_base/datadog_checks/base/checks/win/winpdh_base.py
index bbd1ae358c9a1..574dcda191c09 100644
--- a/datadog_checks_base/datadog_checks/base/checks/win/winpdh_base.py
+++ b/datadog_checks_base/datadog_checks/base/checks/win/winpdh_base.py
@@ -9,7 +9,7 @@
 except ImportError:
     from .winpdh_stub import WinPDHCounter, DATA_TYPE_INT, DATA_TYPE_DOUBLE
 
-from .. import AgentCheck
+from ... import AgentCheck, is_affirmative
 from ...utils.containers import hash_mutable
 
 int_types = [
@@ -34,6 +34,7 @@ def __init__(self, name, init_config, agentConfig, instances, counter_list):
         AgentCheck.__init__(self, name, init_config, agentConfig, instances)
         self._countersettypes = {}
         self._counters = {}
+        self._missing_counters = {}
         self._metrics = {}
         self._tags = {}
         key = None
@@ -95,67 +96,14 @@ def __init__(self, name, init_config, agentConfig, instances, counter_list):
                             else:
                                 self.log.warning("Unknown data type %s" % str(v))
 
-                # list of the metrics.  Each entry is itself an entry,
-                # which is the pdh name, datadog metric name, type, and the
-                # pdh counter object
-
-                for counterset, inst_name, counter_name, dd_name, mtype in counter_list:
-                    m = getattr(self, mtype.lower())
-
-                    precision = datatypes.get(dd_name)
-
-                    try:
-                        obj = WinPDHCounter(
-                            counterset,
-                            counter_name,
-                            self.log,
-                            inst_name,
-                            machine_name=remote_machine,
-                            precision=precision
-                        )
-                    except Exception as e:
-                        self.log.warning("Couldn't create counter {}\\{} due to {}".format(counterset, counter_name, e))
-                        self.log.warning("Datadog Agent will not report {}".format(dd_name))
-                        continue
-
-                    entry = [inst_name, dd_name, m, obj]
-                    self.log.debug("entry: %s" % str(entry))
-                    self._metrics[key].append(entry)
+                self._make_counters(key, (counter_list, (datatypes, remote_machine, False, 'entry')))
 
                 # get any additional metrics in the instance
                 addl_metrics = instance.get('additional_metrics')
                 if addl_metrics is not None:
-                    for counterset, inst_name, counter_name, dd_name, mtype in addl_metrics:
-                        if (
-                            inst_name.lower() == "none" or
-                            len(inst_name) == 0 or
-                            inst_name == "*" or
-                            inst_name.lower() == "all"
-                        ):
-                            inst_name = None
-                        m = getattr(self, mtype.lower())
-
-                        precision = datatypes.get(dd_name)
-
-                        try:
-                            obj = WinPDHCounter(
-                                counterset,
-                                counter_name,
-                                self.log,
-                                inst_name,
-                                machine_name=remote_machine,
-                                precision=precision
-                            )
-                        except Exception as e:
-                            self.log.warning(
-                                "Couldn't create counter {}\\{} due to {}".format(counterset, counter_name, e)
-                            )
-                            self.log.warning("Datadog Agent will not report {}".format(dd_name))
-                            continue
-
-                        entry = [inst_name, dd_name, m, obj]
-                        self.log.debug("additional metric entry: %s" % str(entry))
-                        self._metrics[key].append(entry)
+                    self._make_counters(
+                        key, (addl_metrics, (datatypes, remote_machine, True, 'additional metric entry'))
+                    )
 
         except Exception as e:
             self.log.debug("Exception in PDH init: %s", str(e))
@@ -167,8 +115,16 @@ def __init__(self, name, init_config, agentConfig, instances, counter_list):
     def check(self, instance):
         self.log.debug("PDHBaseCheck: check()")
         key = hash_mutable(instance)
+        cache_counter_instances = is_affirmative(instance.get('cache_counter_instances', True))
+
+        if not cache_counter_instances:
+            for counter, values in list(iteritems(self._missing_counters)):
+                self._make_counters(key, ([counter], values))
+
         for inst_name, dd_name, metric_func, counter in self._metrics[key]:
             try:
+                if not cache_counter_instances:
+                    counter.collect_counters()
                 vals = counter.get_all_values()
                 for instance_name, val in iteritems(vals):
                     tags = []
@@ -182,3 +138,48 @@ def check(self, instance):
             except Exception as e:
                 # don't give up on all of the metrics because one failed
                 self.log.error("Failed to get data for %s %s: %s" % (inst_name, dd_name, str(e)))
+
+    def _make_counters(self, key, counter_data):
+        counter_list, (datatypes, remote_machine, check_instance, message) = counter_data
+
+        # list of the metrics. Each entry is itself an entry,
+        # which is the pdh name, datadog metric name, type, and the
+        # pdh counter object
+        for counterset, inst_name, counter_name, dd_name, mtype in counter_list:
+            if check_instance and self._no_instance(inst_name):
+                inst_name = None
+
+            m = getattr(self, mtype.lower())
+            precision = datatypes.get(dd_name)
+
+            try:
+                obj = WinPDHCounter(
+                    counterset,
+                    counter_name,
+                    self.log,
+                    inst_name,
+                    machine_name=remote_machine,
+                    precision=precision
+                )
+            except Exception as e:
+                self.log.warning('Could not create counter {}\\{} due to {}'.format(counterset, counter_name, e))
+                self.log.warning('Datadog Agent will not report {}'.format(dd_name))
+                self._missing_counters[(counterset, inst_name, counter_name, dd_name, mtype)] = (
+                    datatypes, remote_machine, check_instance, message
+                )
+                continue
+            else:
+                self._missing_counters.pop((counterset, inst_name, counter_name, dd_name, mtype), None)
+
+            entry = [inst_name, dd_name, m, obj]
+            self.log.debug('{}: {}'.format(message, entry))
+            self._metrics[key].append(entry)
+
+    @classmethod
+    def _no_instance(cls, inst_name):
+        return (
+            inst_name.lower() == 'none' or
+            len(inst_name) == 0 or
+            inst_name == '*' or
+            inst_name.lower() == 'all'
+        )
diff --git a/datadog_checks_tests_helper/datadog_test_libs/win/pdh_mocks.py b/datadog_checks_tests_helper/datadog_test_libs/win/pdh_mocks.py
index 63162da7b0470..d9cbe2508aaa5 100644
--- a/datadog_checks_tests_helper/datadog_test_libs/win/pdh_mocks.py
+++ b/datadog_checks_tests_helper/datadog_test_libs/win/pdh_mocks.py
@@ -31,10 +31,12 @@ def pdh_mocks_fixture():
     pdhaddcounter = mock.patch('win32pdh.AddCounter', mock_AddCounter)
     pdhgetformattedcountervalue = mock.patch('win32pdh.GetFormattedCounterValue', mock_GetFormattedCounterValue)
     pdhcollectquerydata = mock.patch('win32pdh.CollectQueryData', mock_CollectQueryData)
+    pdhvalidatepath = mock.patch('win32pdh.ValidatePath', mock_ValidatePath)
 
     yield regqueryvalueex.start(), pdhlookupbyindex.start(), \
         pdhenumobjectitems.start(),  pdhmakecounterpath.start(), \
-        pdhaddcounter.start(), pdhgetformattedcountervalue.start(), pdhcollectquerydata.start()
+        pdhaddcounter.start(), pdhgetformattedcountervalue.start(), pdhcollectquerydata.start(), \
+        pdhvalidatepath.start()
 
     regqueryvalueex.stop()
     pdhlookupbyindex.stop()
@@ -43,6 +45,7 @@ def pdh_mocks_fixture():
     pdhaddcounter.stop()
     pdhgetformattedcountervalue.stop()
     pdhcollectquerydata.stop()
+    pdhvalidatepath.stop()
 
 
 @pytest.fixture
@@ -58,10 +61,12 @@ def pdh_mocks_fixture_bad_perf_strings():
     pdhaddcounter = mock.patch('win32pdh.AddCounter', mock_AddCounter)
     pdhgetformattedcountervalue = mock.patch('win32pdh.GetFormattedCounterValue', mock_GetFormattedCounterValue)
     pdhcollectquerydata = mock.patch('win32pdh.CollectQueryData', mock_CollectQueryData)
+    pdhvalidatepath = mock.patch('win32pdh.ValidatePath', mock_ValidatePath)
 
     yield regqueryvalueex.start(), pdhlookupbyindex.start(), \
         pdhenumobjectitems.start(),  pdhmakecounterpath.start(), \
-        pdhaddcounter.start(), pdhgetformattedcountervalue.start(), pdhcollectquerydata.start()
+        pdhaddcounter.start(), pdhgetformattedcountervalue.start(), pdhcollectquerydata.start(), \
+        pdhvalidatepath.start()
 
     regqueryvalueex.stop()
     pdhlookupbyindex.stop()
@@ -70,6 +75,7 @@ def pdh_mocks_fixture_bad_perf_strings():
     pdhaddcounter.stop()
     pdhgetformattedcountervalue.stop()
     pdhcollectquerydata.stop()
+    pdhvalidatepath.stop()
 
 
 def initialize_pdh_tests(lang=None):
@@ -216,3 +222,7 @@ def mock_QueryValueEx(*args, **kwargs):
 
 def mock_QueryValueExWithRaise(*args, **kwargs):
     raise WindowsError
+
+
+def mock_ValidatePath(p):
+    return 0
diff --git a/dotnetclr/tests/test_bench.py b/dotnetclr/tests/test_bench.py
new file mode 100644
index 0000000000000..29026875af7d7
--- /dev/null
+++ b/dotnetclr/tests/test_bench.py
@@ -0,0 +1,37 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+from datadog_checks.dotnetclr import DotnetclrCheck
+from datadog_test_libs.win.pdh_mocks import pdh_mocks_fixture, initialize_pdh_tests  # noqa: F401
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': True,
+        'host': '.',
+    }
+    check = DotnetclrCheck('dotnetclr', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_no_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': False,
+        'host': '.',
+    }
+    check = DotnetclrCheck('dotnetclr', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
diff --git a/dotnetclr/tests/test_dotnetclr.py b/dotnetclr/tests/test_dotnetclr.py
index 81d5322ce28c5..54120d2e4c42d 100644
--- a/dotnetclr/tests/test_dotnetclr.py
+++ b/dotnetclr/tests/test_dotnetclr.py
@@ -14,10 +14,12 @@
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 MINIMAL_INSTANCE = {
+    'cache_counter_instances': False,
     'host': '.',
 }
 
 INSTANCE_WITH_TAGS = {
+    'cache_counter_instances': False,
     'host': '.',
     'tags': ['tag1', 'another:tag']
 }
diff --git a/dotnetclr/tox.ini b/dotnetclr/tox.ini
index 24cc7a939414a..79ec820d37778 100644
--- a/dotnetclr/tox.ini
+++ b/dotnetclr/tox.ini
@@ -4,25 +4,29 @@ basepython = py27
 envlist =
     dotnetclr
     flake8
+    bench
 
 [testenv]
 usedevelop = true
 platform = win32
-
-[testenv:dotnetclr]
 deps =
     -e../datadog_checks_base[deps]
     ../datadog_checks_tests_helper
     -rrequirements-dev.txt
 commands =
     pip install -r requirements.in
-    pytest -v
+    pytest -v --benchmark-skip
 
 [testenv:flake8]
 skip_install = true
 deps = flake8
 commands = flake8 .
 
+[testenv:bench]
+commands =
+    pip install -r requirements.in
+    pytest --benchmark-only --benchmark-cprofile=tottime
+
 [flake8]
 exclude = .eggs,.tox,build
 max-line-length = 120
diff --git a/exchange_server/tests/test_bench.py b/exchange_server/tests/test_bench.py
new file mode 100644
index 0000000000000..54905389f6171
--- /dev/null
+++ b/exchange_server/tests/test_bench.py
@@ -0,0 +1,37 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+from datadog_checks.exchange_server import ExchangeCheck
+from datadog_test_libs.win.pdh_mocks import pdh_mocks_fixture, initialize_pdh_tests  # noqa: F401
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': True,
+        'host': '.',
+    }
+    check = ExchangeCheck('exchange_server', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_no_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': False,
+        'host': '.',
+    }
+    check = ExchangeCheck('exchange_server', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
diff --git a/exchange_server/tests/test_exchange_server.py b/exchange_server/tests/test_exchange_server.py
index 18ed0919b5abc..05730dd82bf03 100644
--- a/exchange_server/tests/test_exchange_server.py
+++ b/exchange_server/tests/test_exchange_server.py
@@ -14,10 +14,12 @@
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 MINIMAL_INSTANCE = {
+    'cache_counter_instances': False,
     'host': '.',
 }
 
 INSTANCE_WITH_TAGS = {
+    'cache_counter_instances': False,
     'host': '.',
     'tags': ['tag1', 'another:tag']
 }
diff --git a/exchange_server/tox.ini b/exchange_server/tox.ini
index e05e75c36620b..a79c540dfa468 100644
--- a/exchange_server/tox.ini
+++ b/exchange_server/tox.ini
@@ -4,25 +4,29 @@ basepython = py27
 envlist =
     exchange_server
     flake8
+    bench
 
 [testenv]
 usedevelop = true
 platform = win32
-
-[testenv:exchange_server]
 deps =
     -e../datadog_checks_base[deps]
     ../datadog_checks_tests_helper
     -rrequirements-dev.txt
 commands =
     pip install -r requirements.in
-    pytest -v
+    pytest -v --benchmark-skip
 
 [testenv:flake8]
 skip_install = true
 deps = flake8
 commands = flake8 .
 
+[testenv:bench]
+commands =
+    pip install -r requirements.in
+    pytest --benchmark-only --benchmark-cprofile=tottime
+
 [flake8]
 exclude = .eggs,.tox,build
 max-line-length = 120
diff --git a/iis/tests/test_bench.py b/iis/tests/test_bench.py
new file mode 100644
index 0000000000000..d78a6f8004217
--- /dev/null
+++ b/iis/tests/test_bench.py
@@ -0,0 +1,37 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+from datadog_checks.iis import IIS
+from datadog_test_libs.win.pdh_mocks import pdh_mocks_fixture, initialize_pdh_tests  # noqa: F401
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': True,
+        'host': '.',
+    }
+    check = IIS('iis', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_no_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': False,
+        'host': '.',
+    }
+    check = IIS('iis', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
diff --git a/iis/tests/test_iis.py b/iis/tests/test_iis.py
index a6c3b47316119..307dfd3bb8fe7 100644
--- a/iis/tests/test_iis.py
+++ b/iis/tests/test_iis.py
@@ -20,10 +20,12 @@ def Aggregator():
 
 CHECK_NAME = 'iis'
 MINIMAL_INSTANCE = {
+    'cache_counter_instances': False,
     'host': '.',
 }
 
 INSTANCE = {
+    'cache_counter_instances': False,
     'host': '.',
     'sites': ['Default Web Site', 'Exchange Back End', 'Non Existing Website'],
 }
diff --git a/iis/tox.ini b/iis/tox.ini
index 0707c9ac81ce8..712457b0dc4b2 100644
--- a/iis/tox.ini
+++ b/iis/tox.ini
@@ -4,19 +4,18 @@ basepython = py27
 envlist =
     iis
     flake8
+    bench
 
 [testenv]
 usedevelop = true
 platform = win32
-
-[testenv:iis]
 deps =
     -e../datadog_checks_base[deps]
     ../datadog_checks_tests_helper
     -rrequirements-dev.txt
 commands =
     pip install -r requirements.in
-    pytest -v
+    pytest -v --benchmark-skip
 
 [testenv:flake8]
 platform = linux2|darwin|win32
@@ -24,6 +23,11 @@ skip_install = true
 deps = flake8
 commands = flake8 .
 
+[testenv:bench]
+commands =
+    pip install -r requirements.in
+    pytest --benchmark-only --benchmark-cprofile=tottime
+
 [flake8]
 exclude = .eggs,.tox,build
 max-line-length = 120
diff --git a/pdh_check/tests/test_bench.py b/pdh_check/tests/test_bench.py
new file mode 100644
index 0000000000000..a6c848a3b6664
--- /dev/null
+++ b/pdh_check/tests/test_bench.py
@@ -0,0 +1,45 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+from datadog_checks.pdh_check import PDHCheck
+from datadog_test_libs.win.pdh_mocks import pdh_mocks_fixture, initialize_pdh_tests  # noqa: F401
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': True,
+        'countersetname': 'System',
+        'metrics': [
+            ['File Read Operations/sec', 'pdh.system.file_read_per_sec', 'gauge'],
+            ['File Write Bytes/sec', 'pdh.system.file_write_bytes_sec', 'gauge'],
+        ],
+    }
+    check = PDHCheck('pdh_check', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
+
+
+@pytest.mark.usefixtures('pdh_mocks_fixture')
+def test_no_cache(benchmark):
+    initialize_pdh_tests()
+    instance = {
+        'cache_counter_instances': False,
+        'countersetname': 'System',
+        'metrics': [
+            ['File Read Operations/sec', 'pdh.system.file_read_per_sec', 'gauge'],
+            ['File Write Bytes/sec', 'pdh.system.file_write_bytes_sec', 'gauge'],
+        ],
+    }
+    check = PDHCheck('pdh_check', {}, {}, [instance])
+
+    # Run once to get any PDH setup out of the way.
+    check.check(instance)
+
+    benchmark(check.check, instance)
diff --git a/pdh_check/tests/test_pdh_check.py b/pdh_check/tests/test_pdh_check.py
index a800f3a67fba1..e9a022945b29a 100644
--- a/pdh_check/tests/test_pdh_check.py
+++ b/pdh_check/tests/test_pdh_check.py
@@ -14,6 +14,7 @@
 CHECK_NAME = 'pdh_check'
 
 INSTANCE = {
+    'cache_counter_instances': False,
     'countersetname': 'System',
     'metrics': [
         ['File Read Operations/sec', 'pdh.system.file_read_per_sec', 'gauge'],
diff --git a/pdh_check/tox.ini b/pdh_check/tox.ini
index 877f5b8579ae0..74f44ca77e740 100644
--- a/pdh_check/tox.ini
+++ b/pdh_check/tox.ini
@@ -4,12 +4,10 @@ basepython = py27
 envlist =
     pdh_check
     flake8
+    bench
 
 [testenv]
 usedevelop = true
-platform = win32|linux2|darwin
-
-[testenv:pdh_check]
 platform = win32
 deps =
     -e../datadog_checks_base[deps]
@@ -17,13 +15,18 @@ deps =
     -rrequirements-dev.txt
 commands =
     pip install -r requirements.in
-    pytest -v
+    pytest -v --benchmark-skip
 
 [testenv:flake8]
 skip_install = true
 deps = flake8
 commands = flake8 .
 
+[testenv:bench]
+commands =
+    pip install -r requirements.in
+    pytest --benchmark-only --benchmark-cprofile=tottime
+
 [flake8]
 exclude = .eggs,.tox,build
 max-line-length = 120