diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
index 8025a7de3c..0da37079d6 100644
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -46,12 +46,19 @@ if (INTERN_BUILD_ATEN_OPS)
   list(APPEND Caffe2_DEPENDENCY_INCLUDE ${ATen_THIRD_PARTY_INCLUDE})
 endif()
 
+# {Q/X,etc} NPACK support is enabled by default, if none of these options
+# are selected, turn this flag ON to incidate the support is disabled
+set(NNPACK_AND_FAMILY_DISABLED OFF)
+if(NOT (USE_NNPACK OR USE_QNNPACK OR USE_PYTORCH_QNNPACK OR USE_XNNPACK))
+  set(NNPACK_AND_FAMILY_DISABLED ON)
+endif()
+
 # ---[ Caffe2 build
 # Note: the folders that are being commented out have not been properly
 # addressed yet.
 
 # For pthreadpool_new_if_impl. TODO: Remove when threadpools are unitied.
-if (NOT MSVC)
+if (NOT MSVC AND NOT NNPACK_AND_FAMILY_DISABLED)
   IF(NOT TARGET fxdiv)
     SET(FXDIV_BUILD_TESTS OFF CACHE BOOL "")
     SET(FXDIV_BUILD_BENCHMARKS OFF CACHE BOOL "")
@@ -710,7 +717,7 @@ ELSEIF(USE_CUDA)
 ENDIF()
 
 
-if (NOT MSVC)
+if (NOT MSVC AND NOT NNPACK_AND_FAMILY_DISABLED)
   TARGET_LINK_LIBRARIES(torch_cpu PRIVATE fxdiv)
 endif()
 
diff --git a/caffe2/utils/CMakeLists.txt b/caffe2/utils/CMakeLists.txt
index 27aabb1315..3c7845c67d 100644
--- a/caffe2/utils/CMakeLists.txt
+++ b/caffe2/utils/CMakeLists.txt
@@ -36,7 +36,7 @@ list(APPEND Caffe2_CPU_SRCS
 # ---[ threadpool/pthreadpool* is a local modification of the NNPACK
 # pthreadpool with a very similar interface. Neither NNPACK, nor this
 # thread pool supports Windows.
-if (NOT MSVC)
+if (NOT MSVC AND NOT NNPACK_AND_FAMILY_DISABLED)
   add_definitions(-DUSE_INTERNAL_THREADPOOL_IMPL)
   set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS}
           utils/threadpool/pthreadpool.cc