Support same target protocol as HTTPx entry protocol without tls pass…

…through

- introduce `service.beta.kubernetes.io/do-loadbalancer-tls-target-protocol-override` annotation to support same HTTPx target protocol as entry protocol
- default behavior remains unchanged, enabling new annotation affects how forwarding rules will be created
- affects all HTTPx forwarding rule behavior (HTTPS/2/3)

cloud-controller-manager/do/lb_annotations.go

@@ -185,4 +185,9 @@ const (
 	// annDONetwork is the annotation used to specify the network type of the load balancer. Either EXTERNAL or INTERNAL (currently in closed alpha)
 	// are permitted. If no network is provided, then it will default EXTERNAL.
 	annDONetwork = annDOLoadBalancerBase + "network"
+
+	// annDOTLSTargetProtocolOverride is the annotation to specify target protocol override to relay HTTPx requests all the way
+	// through to the load balancer target endpoint instead of redirecting as HTTP. It accepts a boolean value and defaults
+	// to 'false' if unspecified.
+	annDOTLSTargetProtocolOverride = annDOLoadBalancerBase + "tls-target-protocol-override"
 )

cloud-controller-manager/do/loadbalancers.go

@@ -733,13 +733,21 @@ func buildHTTP3ForwardingRule(ctx context.Context, service *v1.Service, godoClie
 		return nil, errors.New("certificate ID is required for HTTP3")
 	}
 
+	// Target protocol defaults to HTTP unless override is configured
+	targetProtocol := protocolHTTP
+	if overrideTargetProtocol, err := getTLSTargetProtocolOverride(service); err != nil {
+		return nil, err
+	} else if overrideTargetProtocol {
+		targetProtocol = protocolHTTP3
+	}
+
 	for _, port := range service.Spec.Ports {
 		if port.Port == int32(http3Port) {
 			return &godo.ForwardingRule{
 				EntryProtocol:  protocolHTTP3,
 				EntryPort:      http3Port,
 				CertificateID:  certificateID,
-				TargetProtocol: protocolHTTP,
+				TargetProtocol: targetProtocol,
 				TargetPort:     int(port.NodePort),
 			}, nil
 		}
@@ -928,6 +936,11 @@ func buildTLSForwardingRule(forwardingRule *godo.ForwardingRule, service *v1.Ser
 		return errors.New("either certificate id should be set or tls pass through enabled, not both")
 	}
 
+	overrideTargetProtocol, err := getTLSTargetProtocolOverride(service)
+	if err != nil {
+		return err
+	}
+
 	if tlsPassThrough {
 		forwardingRule.TlsPassthrough = tlsPassThrough
 		// We don't explicitly set the TargetProtocol here since in buildForwardingRule
@@ -936,7 +949,9 @@ func buildTLSForwardingRule(forwardingRule *godo.ForwardingRule, service *v1.Ser
 		// to match the EntryProtocol.
 	} else {
 		forwardingRule.CertificateID = certificateID
-		forwardingRule.TargetProtocol = protocolHTTP
+		if !overrideTargetProtocol {
+			forwardingRule.TargetProtocol = protocolHTTP
+		}
 	}
 
 	return nil
@@ -1398,6 +1413,14 @@ func getType(service *v1.Service) (string, error) {
 	return name, nil
 }
 
+func getTLSTargetProtocolOverride(service *v1.Service) (bool, error) {
+	tlsTargetProtocolOverride, _, err := getBool(service.Annotations, annDOTLSTargetProtocolOverride)
+	if err != nil {
+		return false, fmt.Errorf("failed to TLS target protocol override configuration setting: %s", err)
+	}
+	return tlsTargetProtocolOverride, nil
+}
+
 func getNetwork(service *v1.Service) (string, error) {
 	network := service.Annotations[annDONetwork]
 	if network == "" {

cloud-controller-manager/do/loadbalancers_test.go

@@ -2161,6 +2161,119 @@ func Test_buildForwardingRules(t *testing.T) {
 			},
 			nil,
 		},
+		{
+			"HTTPS target protocol used if override specified",
+			&v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+					UID:  "abc123",
+					Annotations: map[string]string{
+						annDOTLSPorts:                  "443",
+						annDOCertificateID:             "test-certificate",
+						annDOTLSTargetProtocolOverride: "true",
+					},
+				},
+				Spec: v1.ServiceSpec{
+					Ports: []v1.ServicePort{
+						{
+							Name:     "test",
+							Protocol: "TCP",
+							Port:     int32(443),
+							NodePort: int32(18080),
+						},
+					},
+				},
+			},
+			[]godo.ForwardingRule{
+				{
+					EntryProtocol:  "https",
+					EntryPort:      443,
+					TargetProtocol: "https",
+					TargetPort:     18080,
+					CertificateID:  "test-certificate",
+					TlsPassthrough: false,
+				},
+			},
+			nil,
+		},
+		{
+			"HTTP2 target protocol used if override specified",
+			&v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+					UID:  "abc123",
+					Annotations: map[string]string{
+						annDOHTTP2Ports:                "443",
+						annDOCertificateID:             "test-certificate",
+						annDOTLSTargetProtocolOverride: "true",
+					},
+				},
+				Spec: v1.ServiceSpec{
+					Ports: []v1.ServicePort{
+						{
+							Name:     "test",
+							Protocol: "TCP",
+							Port:     int32(443),
+							NodePort: int32(18080),
+						},
+					},
+				},
+			},
+			[]godo.ForwardingRule{
+				{
+					EntryProtocol:  "http2",
+					EntryPort:      443,
+					TargetProtocol: "http2",
+					TargetPort:     18080,
+					CertificateID:  "test-certificate",
+					TlsPassthrough: false,
+				},
+			},
+			nil,
+		},
+		{
+			"HTTP3 target protocol used if override specified",
+			&v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+					UID:  "abc123",
+					Annotations: map[string]string{
+						annDOHTTP3Port:                 "443",
+						annDOCertificateID:             "test-certificate",
+						annDOTLSTargetProtocolOverride: "true",
+					},
+				},
+				Spec: v1.ServiceSpec{
+					Ports: []v1.ServicePort{
+						{
+							Name:     "test",
+							Protocol: "TCP",
+							Port:     int32(443),
+							NodePort: int32(18080),
+						},
+					},
+				},
+			},
+			[]godo.ForwardingRule{
+				{
+					EntryProtocol:  "https",
+					EntryPort:      443,
+					TargetProtocol: "https",
+					TargetPort:     18080,
+					CertificateID:  "test-certificate",
+					TlsPassthrough: false,
+				},
+				{
+					EntryProtocol:  "http3",
+					EntryPort:      443,
+					TargetProtocol: "http3",
+					TargetPort:     18080,
+					CertificateID:  "test-certificate",
+					TlsPassthrough: false,
+				},
+			},
+			nil,
+		},
 	}
 
 	for _, test := range testcases {
@@ -6250,6 +6363,82 @@ func Test_getNetwork(t *testing.T) {
 	}
 }
 
+func Test_getTLSTargetProtocolOverride(t *testing.T) {
+	testcases := []struct {
+		name     string
+		service  *v1.Service
+		wantErr  bool
+		expected bool
+	}{
+		{
+			name: "no value defaults to false",
+			service: &v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        "test",
+					UID:         "abc123",
+					Annotations: map[string]string{},
+				},
+			},
+			wantErr:  false,
+			expected: false,
+		},
+		{
+			name: "annotation set to false",
+			service: &v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+					UID:  "abc123",
+					Annotations: map[string]string{
+						annDOTLSTargetProtocolOverride: "false",
+					},
+				},
+			},
+			wantErr:  false,
+			expected: false,
+		},
+		{
+			name: "annotation set to true",
+			service: &v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+					UID:  "abc123",
+					Annotations: map[string]string{
+						annDOTLSTargetProtocolOverride: "true",
+					},
+				},
+			},
+			wantErr:  false,
+			expected: true,
+		},
+		{
+			name: "illegal value",
+			service: &v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+					UID:  "abc123",
+					Annotations: map[string]string{
+						annDOTLSTargetProtocolOverride: "abcd",
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, test := range testcases {
+		t.Run(test.name, func(t *testing.T) {
+			option, err := getTLSTargetProtocolOverride(test.service)
+			if test.wantErr != (err != nil) {
+				t.Errorf("got error %q, want error: %t", err, test.wantErr)
+			}
+
+			if option != test.expected {
+				t.Fatalf("got %v, want %v", option, test.expected)
+			}
+		})
+	}
+}
+
 func Test_buildFirewall(t *testing.T) {
 	testcases := []struct {
 		name             string