diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp
index 1c7799e7696ea3..4d1be1e5934312 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -376,8 +376,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
                 auto column_name = inverted_index.columns[0];
                 auto column_idx = output_rowset_schema->field_index(column_name);
                 if (column_idx < 0) {
-                    column_idx =
-                            output_rowset_schema->field_index(inverted_index.column_unique_ids[0]);
+                    if (!inverted_index.column_unique_ids.empty()) {
+                        column_idx = output_rowset_schema->field_index(
+                                inverted_index.column_unique_ids[0]);
+                    }
                     if (column_idx < 0) {
                         LOG(WARNING) << "referenced column was missing. "
                                      << "[column=" << column_name