diff --git a/src/tl/ir.cc b/src/tl/ir.cc
index 24dc0ad753b5..edffb3147b4e 100644
--- a/src/tl/ir.cc
+++ b/src/tl/ir.cc
@@ -74,10 +74,10 @@ ForFrame PipelinedFor(
     ICHECK(n == 1);
     Map<String, ObjectRef> anno;
     if (num_stages > 0) anno.Set("num_stages", PrimExpr(num_stages));
-    anno.Set("tl_pipeline_order", order);
-    anno.Set("tl_pipeline_stage", stages);
-    anno.Set("tl_pipeline_sync", sync);
-    anno.Set("tl_pipeline_group", groups);
+    if (order.size() > 0) anno.Set("tl_pipeline_order", order);
+    if (stages.size() > 0) anno.Set("tl_pipeline_stage", stages);
+    if (sync.size() > 0) anno.Set("tl_pipeline_sync", sync);
+    if (groups.size() > 0) anno.Set("tl_pipeline_group", groups);
     body = For(vars[0], doms[0]->min, doms[0]->extent, ForKind::kSerial, std::move(body),
                /*thread_binding=*/NullOpt, /*annotations=*/anno);
     return body;