diff --git a/include/tvm/ir/attrs.h b/include/tvm/ir/attrs.h
index 4cdf8c5cbe94..d20ba4f95456 100644
--- a/include/tvm/ir/attrs.h
+++ b/include/tvm/ir/attrs.h
@@ -337,6 +337,18 @@ struct AttrInitEntry {
   T* value_;
   // whether the value is missing.
   bool value_missing_{true};
+
+  AttrInitEntry() = default;
+
+  AttrInitEntry(AttrInitEntry&& other) {
+    type_key_ = other.type_key_;
+    key_ = other.key_;
+    value_ = other.value_;
+    value_missing_ = other.value_missing_;
+    // avoid unexpected throw
+    other.value_missing_ = false;
+  }
+
   // If the value is still missing in destruction time throw an error.
   ~AttrInitEntry() DMLC_THROW_EXCEPTION {
     if (value_missing_) {
@@ -463,7 +475,7 @@ class AttrInitVisitor {
     } else {
       opt.value_missing_ = true;
     }
-    return opt;
+    return std::move(opt);
   }
 
  private: