Make SurfaceTool.generate_normals() behave consistently with smoothing groups

doc/classes/SurfaceTool.xml

@@ -133,7 +133,7 @@
 			<description>
 				Generates normals from vertices so you do not have to do it manually. If [param flip] is [code]true[/code], the resulting normals will be inverted. [method generate_normals] should be called [i]after[/i] generating geometry and [i]before[/i] committing the mesh using [method commit] or [method commit_to_arrays]. For correct display of normal-mapped surfaces, you will also have to generate tangents using [method generate_tangents].
 				[b]Note:[/b] [method generate_normals] only works if the primitive type to be set to [constant Mesh.PRIMITIVE_TRIANGLES].
-				[b]Note:[/b] [method generate_normals] takes smooth groups into account. If you don't specify any smooth group for each vertex, [method generate_normals] will smooth normals for you.
+				[b]Note:[/b] [method generate_normals] takes smooth groups into account. To generate smooth normals, set the smooth group to a value greater than or equal to [code]0[/code] using [method set_smooth_group] or leave the smooth group at the default of [code]0[/code]. To generate flat normals, set the smooth group to [code]-1[/code] using [method set_smooth_group] prior to adding vertices.
 			</description>
 		</method>
 		<method name="generate_tangents">
@@ -241,7 +241,7 @@
 			<return type="void" />
 			<param index="0" name="index" type="int" />
 			<description>
-				Specifies whether the current vertex (if using only vertex arrays) or current index (if also using index arrays) should use smooth normals for normal calculation.
+				Specifies the smooth group to use for the [i]next[/i] vertex. If this is never called, all vertices will have the default smooth group of [code]0[/code] and will be smoothed with adjacent vertices of the same group. To produce a mesh with flat normals, set the smooth group to [code]-1[/code].
 			</description>
 		</method>
 		<method name="set_tangent">

scene/resources/surface_tool.cpp

@@ -146,6 +146,25 @@ uint32_t SurfaceTool::VertexHasher::hash(const Vertex &p_vtx) {
 	return h;
 }
 
+bool SurfaceTool::SmoothGroupVertex::operator==(const SmoothGroupVertex &p_vertex) const {
+	if (vertex != p_vertex.vertex) {
+		return false;
+	}
+
+	if (smooth_group != p_vertex.smooth_group) {
+		return false;
+	}
+
+	return true;
+}
+
+uint32_t SurfaceTool::SmoothGroupVertexHasher::hash(const SmoothGroupVertex &p_vtx) {
+	uint32_t h = hash_djb2_buffer((const uint8_t *)&p_vtx.vertex, sizeof(real_t) * 3);
+	h = hash_murmur3_one_32(p_vtx.smooth_group, h);
+	h = hash_fmix32(h);
+	return h;
+}
+
 uint32_t SurfaceTool::TriangleHasher::hash(const int *p_triangle) {
 	int t0 = p_triangle[0];
 	int t1 = p_triangle[1];
@@ -1152,7 +1171,7 @@ void SurfaceTool::generate_normals(bool p_flip) {
 
 	ERR_FAIL_COND((vertex_array.size() % 3) != 0);
 
-	HashMap<Vertex, Vector3, VertexHasher> vertex_hash;
+	HashMap<SmoothGroupVertex, Vector3, SmoothGroupVertexHasher> smooth_hash;
 
 	for (uint32_t vi = 0; vi < vertex_array.size(); vi += 3) {
 		Vertex *v = &vertex_array[vi];
@@ -1165,21 +1184,28 @@ void SurfaceTool::generate_normals(bool p_flip) {
 		}
 
 		for (int i = 0; i < 3; i++) {
-			Vector3 *lv = vertex_hash.getptr(v[i]);
-			if (!lv) {
-				vertex_hash.insert(v[i], normal);
+			// Add face normal to smooth vertex influence if vertex is member of a smoothing group
+			if (v[i].smooth_group != UINT32_MAX) {
+				Vector3 *lv = smooth_hash.getptr(v[i]);
+				if (!lv) {
+					smooth_hash.insert(v[i], normal);
+				} else {
+					(*lv) += normal;
+				}
 			} else {
-				(*lv) += normal;
+				v[i].normal = normal;
 			}
 		}
 	}
 
 	for (Vertex &vertex : vertex_array) {
-		Vector3 *lv = vertex_hash.getptr(vertex);
-		if (!lv) {
-			vertex.normal = Vector3();
-		} else {
-			vertex.normal = lv->normalized();
+		if (vertex.smooth_group != UINT32_MAX) {
+			Vector3 *lv = smooth_hash.getptr(vertex);
+			if (!lv) {
+				vertex.normal = Vector3();
+			} else {
+				vertex.normal = lv->normalized();
+			}
 		}
 	}
 

scene/resources/surface_tool.h

@@ -97,6 +97,21 @@ class SurfaceTool : public RefCounted {
 		static _FORCE_INLINE_ uint32_t hash(const Vertex &p_vtx);
 	};
 
+	struct SmoothGroupVertex {
+		Vector3 vertex;
+		uint32_t smooth_group = 0;
+		bool operator==(const SmoothGroupVertex &p_vertex) const;
+
+		SmoothGroupVertex(const Vertex &p_vertex) {
+			vertex = p_vertex.vertex;
+			smooth_group = p_vertex.smooth_group;
+		};
+	};
+
+	struct SmoothGroupVertexHasher {
+		static _FORCE_INLINE_ uint32_t hash(const SmoothGroupVertex &p_vtx);
+	};
+
 	struct TriangleHasher {
 		static _FORCE_INLINE_ uint32_t hash(const int *p_triangle);
 		static _FORCE_INLINE_ bool compare(const int *p_lhs, const int *p_rhs);