refactor!: Removing deprecated InputRange, op_precision and input_shapes

APIs

BREAKING CHANGE: This removes the InputRange Class and op_precision and
input shape fields which were deprecated in TRTorch v0.4.0

Signed-off-by: Naren Dasan <[email protected]>
Signed-off-by: Naren Dasan <[email protected]>

Author: narendasan

cpp/include/trtorch/trtorch.h

@@ -506,64 +506,6 @@ struct TRTORCH_API CompileSpec {
     bool explicit_set_dtype;
   };
 
-  /**
-   * @brief A struct to hold an input range (used by TensorRT Optimization
-   * profile)
-   *
-   * This struct can either hold a single vector representing an input shape,
-   * signifying a static input shape or a set of three input shapes representing
-   * the min, optiminal and max input shapes allowed for the engine.
-   */
-  struct TRTORCH_API InputRange {
-    /// Minimum acceptable input size into the engine
-    std::vector<int64_t> min;
-    /// Optimal input size into the engine (gets best performace)
-    std::vector<int64_t> opt;
-    /// Maximum acceptable input size into the engine
-    std::vector<int64_t> max;
-    /**
-     * @brief Construct a new Input Range object for static input size from
-     * vector
-     *
-     * @param opt
-     */
-    [[deprecated("trtorch::CompileSpec::InputRange is being deprecated in favor of trtorch::CompileSpec::Input. trtorch::CompileSpec::InputRange will be removed in TRTorch v0.5.0")]] InputRange(
-        std::vector<int64_t> opt);
-    /**
-     * @brief Construct a new Input Range object static input size from
-     * c10::ArrayRef (the type produced by tensor.sizes())
-     *
-     * @param opt
-     */
-    [[deprecated("trtorch::CompileSpec::InputRange is being deprecated in favor of trtorch::CompileSpec::Input. trtorch::CompileSpec::InputRange will be removed in TRTorch v0.5.0")]] InputRange(
-        c10::ArrayRef<int64_t> opt);
-    /**
-     * @brief Construct a new Input Range object dynamic input size from vectors
-     * for min, opt, and max supported sizes
-     *
-     * @param min
-     * @param opt
-     * @param max
-     */
-    [[deprecated("trtorch::CompileSpec::InputRange is being deprecated in favor of trtorch::CompileSpec::Input. trtorch::CompileSpec::InputRange will be removed in TRTorch v0.5.0")]] InputRange(
-        std::vector<int64_t> min,
-        std::vector<int64_t> opt,
-        std::vector<int64_t> max);
-    /**
-     * @brief Construct a new Input Range object dynamic input size from
-     * c10::ArrayRef (the type produced by tensor.sizes()) for min, opt, and max
-     * supported sizes
-     *
-     * @param min
-     * @param opt
-     * @param max
-     */
-    [[deprecated("trtorch::CompileSpec::InputRange is being deprecated in favor of trtorch::CompileSpec::Input. trtorch::CompileSpec::InputRange will be removed in TRTorch v0.5.0")]] InputRange(
-        c10::ArrayRef<int64_t> min,
-        c10::ArrayRef<int64_t> opt,
-        c10::ArrayRef<int64_t> max);
-  };
-
   /**
    * @brief A struct to hold fallback info
    */
@@ -596,18 +538,6 @@ struct TRTORCH_API CompileSpec {
     TorchFallback(bool enabled, uint64_t min_size) : enabled(enabled), min_block_size(min_size) {}
   };
 
-  /**
-   * @brief Construct a new Extra Info object from input ranges.
-   * Each entry in the vector represents a input and should be provided in call
-   * order.
-   *
-   * Use this constructor if you want to use dynamic shape
-   *
-   * @param input_ranges
-   */
-  [[deprecated("trtorch::CompileSpec::CompileSpec(std::vector<InputRange> input_ranges) is being deprecated in favor of trtorch::CompileSpec::CompileSpec(std::vector<Input> inputs). Please use CompileSpec(std::vector<Input> inputs). trtorch::CompileSpec::CompileSpec(std::vector<InputRange> input_ranges) will be removed in TRTorch v0.5.0")]] CompileSpec(
-      std::vector<InputRange> input_ranges)
-      : input_ranges(std::move(input_ranges)) {}
   /**
    * @brief Construct a new Extra Info object
    * Convienence constructor to set fixed input size from vectors describing
@@ -657,24 +587,6 @@ struct TRTORCH_API CompileSpec {
    */
   std::vector<Input> inputs;
 
-  /**
-   * Sizes for inputs to engine, can either be a single size or a range
-   * defined by Min, Optimal, Max sizes
-   *
-   * Order is should match call order
-   */
-  [[deprecated(
-      "trtorch::CompileSpec::input_ranges is being deprecated in favor of trtorch::CompileSpec::inputs. trtorch::CompileSpec::input_ranges will be removed in TRTorch v0.5.0")]] std::
-      vector<InputRange>
-          input_ranges;
-
-  /**
-   * Default operating precision for the engine
-   */
-  [[deprecated(
-      "trtorch::CompileSpec::op_precision is being deprecated in favor of trtorch::CompileSpec::enabled_precisions, a set of all enabled precisions to use during compilation, trtorch::CompileSpec::op_precision will be removed in TRTorch v0.5.0")]] DataType
-      op_precision = DataType::kFloat;
-
   /**
    * @brief The set of precisions TensorRT is allowed to use for kernels during compilation
    *

cpp/src/compile_spec.cpp

@@ -144,30 +144,6 @@ CompileSpec::Device::DeviceType::DeviceType(c10::DeviceType t) {
   value = DeviceType::kGPU;
 }
 
-CompileSpec::InputRange::InputRange(std::vector<int64_t> opt) {
-  this->opt = opt;
-  this->min = opt;
-  this->max = opt;
-}
-
-CompileSpec::InputRange::InputRange(c10::IntArrayRef opt) {
-  this->opt = core::util::toVec(opt);
-  this->min = core::util::toVec(opt);
-  this->max = core::util::toVec(opt);
-}
-
-CompileSpec::InputRange::InputRange(std::vector<int64_t> min, std::vector<int64_t> opt, std::vector<int64_t> max) {
-  this->opt = opt;
-  this->min = min;
-  this->max = max;
-}
-
-CompileSpec::InputRange::InputRange(c10::IntArrayRef min, c10::IntArrayRef opt, c10::IntArrayRef max) {
-  this->opt = core::util::toVec(opt);
-  this->min = core::util::toVec(min);
-  this->max = core::util::toVec(max);
-}
-
 CompileSpec::CompileSpec(std::vector<c10::ArrayRef<int64_t>> fixed_sizes) {
   for (auto in : fixed_sizes) {
     inputs.push_back(Input(in));
@@ -289,22 +265,10 @@ CompileSpec::Input::Input(
 
 /* ==========================================*/
 
-core::ir::Input to_internal_input(CompileSpec::InputRange& i) {
-  return core::ir::Input(i.min, i.opt, i.max);
-}
-
 core::ir::Input to_internal_input(CompileSpec::Input& i) {
   return core::ir::Input(i.min_shape, i.opt_shape, i.max_shape, toTRTDataType(i.dtype), toTRTTensorFormat(i.format));
 }
 
-std::vector<core::ir::Input> to_vec_internal_inputs(std::vector<CompileSpec::InputRange>& external) {
-  std::vector<core::ir::Input> internal;
-  for (auto range : external) {
-    internal.push_back(to_internal_input(range));
-  }
-  return internal;
-}
-
 std::vector<core::ir::Input> to_vec_internal_inputs(std::vector<CompileSpec::Input>& external) {
   std::vector<core::ir::Input> internal;
   for (auto range : external) {
@@ -328,24 +292,9 @@ core::runtime::CudaDevice to_internal_cuda_device(CompileSpec::Device device) {
 
 core::CompileSpec to_internal_compile_spec(CompileSpec external) {
   core::CompileSpec internal(to_vec_internal_inputs(external.inputs));
-  if (external.input_ranges.size() > 0 && external.inputs.size() > 0) {
-    TRTORCH_THROW_ERROR(
-        "Saw both input specs listed for inputs and input_ranges in CompileSpec. input_ranges is deprecated and will be removed in v0.5.0. Please port forward to using inputs");
-  } else if (external.input_ranges.size() > 0) {
-    internal = core::CompileSpec(to_vec_internal_inputs(external.input_ranges));
-  } else {
-    TRTORCH_CHECK(external.inputs.size() > 0, "Compilation requires at least one input specification");
-    internal = core::CompileSpec(to_vec_internal_inputs(external.inputs));
-  }
 
-  if (external.enabled_precisions.size() == 1 &&
-      toTRTDataType(*external.enabled_precisions.begin()) == nvinfer1::DataType::kFLOAT &&
-      toTRTDataType(external.op_precision) != nvinfer1::DataType::kFLOAT) {
-    internal.convert_info.engine_settings.enabled_precisions.insert(toTRTDataType(external.op_precision));
-  } else {
-    for (auto p : external.enabled_precisions) {
-      internal.convert_info.engine_settings.enabled_precisions.insert(toTRTDataType(p));
-    }
+  for (auto p : external.enabled_precisions) {
+    internal.convert_info.engine_settings.enabled_precisions.insert(toTRTDataType(p));
   }
 
   /* We want default behavior for types to match PyTorch, so in the case the user did not explicitly set the dtype for

examples/int8/training/vgg16/test_qat.py

@@ -81,7 +81,7 @@ def test(model, dataloader, crit):
 import trtorch
 # trtorch.logging.set_reportable_log_level(trtorch.logging.Level.Debug)
 compile_settings = {
-"input_shapes": [[1, 3, 32, 32]],
+"inputs": [trtorch.Input([1, 3, 32, 32])],
 "op_precision": torch.int8 # Run with FP16
 }
 new_mod = torch.jit.load('trained_vgg16_qat.jit.pt')

py/trtorch/_compile_spec.py

@@ -157,22 +157,11 @@ def _parse_torch_fallback(fallback_info: Dict[str, Any]) -> trtorch._C.TorchFall
 
 def _parse_compile_spec(compile_spec: Dict[str, Any]) -> trtorch._C.CompileSpec:
     info = trtorch._C.CompileSpec()
-    if "input_shapes" not in compile_spec and "inputs" not in compile_spec:
+    if "inputs" not in compile_spec:
         raise KeyError(
             "Module input definitions are requried to compile module. Provide a list of trtorch.Input keyed to \"inputs\" in the compile spec"
         )
 
-    if "input_shapes" in compile_spec and "inputs" in compile_spec:
-        raise KeyError(
-            "Found both key \"input_shapes\", and \"inputs\" in compile spec, please port forward to using only \"inputs\""
-        )
-
-    if "input_shapes" in compile_spec:
-        warnings.warn(
-            "Key \"input_shapes\" is deprecated in favor of \"inputs\". Support for \"input_shapes\" will be removed in TRTorch v0.5.0",
-            DeprecationWarning)
-        info.inputs = _parse_input_ranges(compile_spec["input_shapes"])
-
     if "inputs" in compile_spec:
         info.inputs = [i._to_internal() for i in compile_spec["inputs"]]
 
@@ -181,12 +170,6 @@ def _parse_compile_spec(compile_spec: Dict[str, Any]) -> trtorch._C.CompileSpec:
             "Found both key \"op_precision\", and \"enabled_precisions\" in compile spec, please port forward to using only \"enabled_precisions\""
         )
 
-    if "op_precision" in compile_spec:
-        warnings.warn(
-            "Key \"op_precision\" is being deprecated in favor of \"enabled_precision\" which expects a set of precisions to be enabled during compilation (FP32 will always be enabled), Support for \"op_precision\" will be removed in TRTorch v0.5.0",
-            DeprecationWarning)
-        info.enabled_precisions = _parse_enabled_precisions(compile_spec["op_precision"])
-
     if "enabled_precisions" in compile_spec:
         info.enabled_precisions = _parse_enabled_precisions(compile_spec["enabled_precisions"])
         # We want default behavior to match PyTorch, so in the case the user did not explicitly set the dtype for inputs they

tests/py/test_api.py

@@ -13,38 +13,6 @@ def setUp(self):
         self.traced_model = torch.jit.trace(self.model, [self.input])
         self.scripted_model = torch.jit.script(self.model)
 
-    def test_compile_traced_deprecated(self):
-        compile_spec = {
-            "input_shapes": [self.input.shape],
-            "device": {
-                "device_type": trtorch.DeviceType.GPU,
-                "gpu_id": 0,
-                "dla_core": 0,
-                "allow_gpu_fallback": False,
-                "disable_tf32": False
-            }
-        }
-
-        trt_mod = trtorch.compile(self.traced_model, compile_spec)
-        same = (trt_mod(self.input) - self.traced_model(self.input)).abs().max()
-        self.assertTrue(same < 2e-3)
-
-    def test_compile_script_deprecated(self):
-        compile_spec = {
-            "input_shapes": [self.input.shape],
-            "device": {
-                "device_type": trtorch.DeviceType.GPU,
-                "gpu_id": 0,
-                "dla_core": 0,
-                "allow_gpu_fallback": False,
-                "disable_tf32": False
-            }
-        }
-
-        trt_mod = trtorch.compile(self.scripted_model, compile_spec)
-        same = (trt_mod(self.input) - self.scripted_model(self.input)).abs().max()
-        self.assertTrue(same < 2e-3)
-
     def test_compile_traced(self):
         compile_spec = {
             "inputs": [trtorch.Input(self.input.shape, dtype=torch.float, format=torch.contiguous_format)],