diff --git a/internal/gatewayapi/address.go b/internal/gatewayapi/address.go
new file mode 100644
index 00000000000..424e1da29d2
--- /dev/null
+++ b/internal/gatewayapi/address.go
@@ -0,0 +1,40 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package gatewayapi
+
+import (
+	"net/netip"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"sigs.k8s.io/gateway-api/apis/v1beta1"
+)
+
+var _ AddressesTranslator = (*Translator)(nil)
+
+type AddressesTranslator interface {
+	ProcessAddresses(gateways []*GatewayContext, xdsIR XdsIRMap, infraIR InfraIRMap, resources *Resources)
+}
+
+func (t *Translator) ProcessAddresses(gateways []*GatewayContext, xdsIR XdsIRMap, infraIR InfraIRMap, resources *Resources) {
+	for _, gateway := range gateways {
+		// Infra IR already exist
+		irKey := irStringKey(gateway.Gateway)
+		gwInfraIR := infraIR[irKey]
+
+		ipAddr := sets.Set[string]{}
+
+		for _, addr := range gateway.Spec.Addresses {
+			switch *addr.Type {
+			case v1beta1.IPAddressType:
+				if _, err := netip.ParseAddr(addr.Value); err == nil {
+					ipAddr.Insert(addr.Value)
+				}
+			}
+		}
+
+		gwInfraIR.Proxy.Addresses = sets.List(ipAddr)
+	}
+}
diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go
index 41309113389..bd6865f1bbd 100644
--- a/internal/gatewayapi/listener.go
+++ b/internal/gatewayapi/listener.go
@@ -133,25 +133,3 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap
 		}
 	}
 }
-
-func newProxyAddresses(as []v1beta1.GatewayAddress) []ir.ProxyAddress {
-	if len(as) == 0 {
-		return nil
-	}
-
-	out := make([]ir.ProxyAddress, len(as))
-	for i, a := range as {
-		if a.Type == nil {
-			continue
-		}
-		switch *a.Type {
-		case v1beta1.HostnameAddressType:
-			out[i] = ir.ProxyHostname(a.Value)
-		case v1beta1.IPAddressType:
-			out[i] = ir.ProxyIPAddress(a.Value)
-		case v1beta1.NamedAddressType:
-			out[i] = ir.ProxyNamedAddress(a.Value)
-		}
-	}
-	return out
-}
diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go
index 8bfd78d69be..75295b450a1 100644
--- a/internal/gatewayapi/translator.go
+++ b/internal/gatewayapi/translator.go
@@ -46,6 +46,7 @@ type TranslatorManager interface {
 
 	RoutesTranslator
 	ListenersTranslator
+	AddressesTranslator
 	FiltersTranslator
 }
 
@@ -119,6 +120,9 @@ func (t *Translator) Translate(resources *Resources) *TranslateResult {
 	// Process all Listeners for all relevant Gateways.
 	t.ProcessListeners(gateways, xdsIR, infraIR, resources)
 
+	// Process all Addresses for all relevant Gateways.
+	t.ProcessAddresses(gateways, xdsIR, infraIR, resources)
+
 	// Process all relevant HTTPRoutes.
 	httpRoutes := t.ProcessHTTPRoutes(resources.HTTPRoutes, gateways, resources, xdsIR)
 
diff --git a/internal/infrastructure/kubernetes/proxy_service.go b/internal/infrastructure/kubernetes/proxy_service.go
index f71d32955ac..f9bd4a0828d 100644
--- a/internal/infrastructure/kubernetes/proxy_service.go
+++ b/internal/infrastructure/kubernetes/proxy_service.go
@@ -50,9 +50,14 @@ func (i *Infra) expectedProxyService(infra *ir.Infra) (*corev1.Service, error) {
 	if envoyServiceConfig.Annotations != nil {
 		annotations = envoyServiceConfig.Annotations
 	}
+
+	// Set the spec of envoy gateway service
 	serviceSpec := expectedServiceSpec(envoyServiceConfig.Type)
 	serviceSpec.Ports = ports
 	serviceSpec.Selector = getSelector(labels).MatchLabels
+	if len(infra.Proxy.Addresses) > 0 {
+		serviceSpec.ExternalIPs = infra.Proxy.Addresses
+	}
 
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
diff --git a/internal/ir/infra.go b/internal/ir/infra.go
index 86780cdc2c5..09249b8aa31 100644
--- a/internal/ir/infra.go
+++ b/internal/ir/infra.go
@@ -36,10 +36,9 @@ type ProxyInfra struct {
 	Config *v1alpha1.EnvoyProxy
 	// Listeners define the listeners exposed by the proxy infrastructure.
 	Listeners []ProxyListener
-
 	// Addresses contain the external addresses this gateway has been
 	// requested to be available at.
-	Addresses []ProxyAddress
+	Addresses []string
 }
 
 // InfraMetadata defines metadata for the managed proxy infrastructure.
@@ -59,22 +58,6 @@ type ProxyListener struct {
 	Ports []ListenerPort
 }
 
-type ProxyAddress interface {
-	isProxyAddress()
-}
-
-type ProxyHostname string
-
-func (ProxyHostname) isProxyAddress() {}
-
-type ProxyIPAddress string
-
-func (ProxyIPAddress) isProxyAddress() {}
-
-type ProxyNamedAddress string
-
-func (ProxyNamedAddress) isProxyAddress() {}
-
 // ListenerPort defines a network port of a listener.
 // +k8s:deepcopy-gen=true
 type ListenerPort struct {
diff --git a/internal/status/gateway.go b/internal/status/gateway.go
index ae644a66ed7..1ee365155f7 100644
--- a/internal/status/gateway.go
+++ b/internal/status/gateway.go
@@ -50,6 +50,10 @@ func UpdateGatewayStatusProgrammedCondition(gw *gwapiv1b1.Gateway, svc *corev1.S
 			}
 		}
 
+		for _, eip := range svc.Spec.ExternalIPs {
+			addresses = append(addresses, eip)
+		}
+
 		var gwAddresses []gwapiv1b1.GatewayAddress
 		for i := range addresses {
 			addr := gwapiv1b1.GatewayAddress{
@@ -68,11 +72,8 @@ func UpdateGatewayStatusProgrammedCondition(gw *gwapiv1b1.Gateway, svc *corev1.S
 		}
 
 		gw.Status.Addresses = gwAddresses
-		if len(gw.Status.Addresses) == 0 {
-			gw.Status.Addresses = gw.Spec.Addresses
-		}
 	} else {
-		gw.Status.Addresses = gw.Spec.Addresses
+		gw.Status.Addresses = nil
 	}
 	// Update the programmed condition.
 	gw.Status.Conditions = MergeConditions(gw.Status.Conditions, computeGatewayProgrammedCondition(gw, deployment))