From b5f87ceed8e7f4b11ba2df1d20ef72cd46e7ec84 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Fri, 1 Mar 2024 03:26:08 -0500 Subject: [PATCH 1/5] Add a deployment that uploads/updates the amphora image It does: - create a pod with 2 containers: - one container contains the images and copies them to a volume - one container runs an Apache server and serves the amphora images - use gophercloud to ensure that the images are correctly uploaded to glance Right now, it requires a custom URL for the amphora image container: oc patch octavia octavia --type=merge --patch '{"spec":{"amphoraImageContainerImage":"registry-proxy.engineering.redhat.com/rh-osbs/rhosp-dev-preview-octavia-amphora-image:18.0.0"}}' --- api/bases/octavia.openstack.org_octavias.yaml | 67 + api/v1beta1/octavia_types.go | 29 + api/v1beta1/octavia_webhook.go | 1 + api/v1beta1/octaviaapi_types.go | 3 + api/v1beta1/zz_generated.deepcopy.go | 1 + .../bases/octavia.openstack.org_octavias.yaml | 67 + config/default/manager_default_images.yaml | 2 + controllers/octavia_controller.go | 94 + main.go | 1 + pkg/octavia/client.go | 60 +- pkg/octavia/image_upload_deployment.go | 174 ++ pkg/octavia/images.go | 237 +++ templates/octavia/config/httpd.conf | 346 ++++ templates/octavia/config/mime.types | 1828 +++++++++++++++++ 14 files changed, 2909 insertions(+), 1 deletion(-) create mode 100644 pkg/octavia/image_upload_deployment.go create mode 100644 pkg/octavia/images.go create mode 100644 templates/octavia/config/httpd.conf create mode 100644 templates/octavia/config/mime.types diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index 21f73c93..747ed061 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -68,6 +68,12 @@ spec: - name type: object type: array + amphoraImageContainerImage: + description: Octavia Container Image URL + type: string + apacheContainerImage: + description: Apache Container Image URL + type: string customServiceConfig: default: '# add your customization here' description: CustomServiceConfig - customize the service config using @@ -1078,10 +1084,63 @@ spec: description: RabbitMQ instance name Needed to request a transportURL that is created and used in Octavia type: string + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined in + spec.resourceClaims, that are used by this container. \n This + is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be set + for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims + of the Pod where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object secret: description: Secret containing OpenStack password information for octavia's keystone password; no longer used for database password type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string serviceUser: default: octavia description: ServiceUser - service user name @@ -1097,11 +1156,19 @@ spec: description: LoadBalancerSSHPubKey - The name of the ConfigMap containing the pubilc key for connecting to the amphorae via SSH type: string + tenantName: + default: service + description: TenantName - the name of the OpenStack tenant that controls + the Octavia resources + type: string required: + - apacheContainerImage - databaseInstance - octaviaAPI - rabbitMqClusterName - secret + - serviceAccount + - tenantName type: object status: description: OctaviaStatus defines the observed state of Octavia diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index 52982312..1afa5870 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -19,6 +19,7 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -36,6 +37,10 @@ const ( // OctaviaWorkerContainerImage is the fall-back container image for OctaviaWorker OctaviaWorkerContainerImage = "quay.io/podified-antelope-centos9/openstack-octavia-worker:current-podified" + + // ApacheImage - default fall-back image for Apache + // TODO(gthiemonge) need a more recent image? + ApacheContainerImage = "registry.redhat.io/rhel8/httpd-24:latest" ) // OctaviaSpec defines the desired state of Octavia @@ -170,6 +175,28 @@ type OctaviaSpecBase struct { // +kubebuilder:default={} // AmphoraCustomFlavors - User-defined flavors for Octavia AmphoraCustomFlavors []OctaviaAmphoraFlavor `json:"amphoraCustomFlavors,omitempty"` + + // +kubebuilder:validation:Required + // ServiceAccount - service account name used internally to provide Octavia services the default SA name + ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // Resources - Compute Resources required by this service (Limits/Requests). + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +kubebuilder:validation:Optional + // Octavia Container Image URL + AmphoraImageContainerImage string `json:"amphoraImageContainerImage"` + + // +kubebuilder:validation:Required + // Apache Container Image URL + ApacheContainerImage string `json:"apacheContainerImage"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=service + // TenantName - the name of the OpenStack tenant that controls the Octavia resources + TenantName string `json:"tenantName"` } // PasswordSelector to identify the DB and AdminUser password from the Secret @@ -292,6 +319,8 @@ func SetupDefaults() { HousekeepingContainerImageURL: util.GetEnvVar("RELATED_IMAGE_OCTAVIA_HOUSEKEEPING_IMAGE_URL_DEFAULT", OctaviaHousekeepingContainerImage), HealthManagerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_OCTAVIA_HEALTHMANAGER_IMAGE_URL_DEFAULT", OctaviaHealthManagerContainerImage), WorkerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_OCTAVIA_WORKER_IMAGE_URL_DEFAULT", OctaviaWorkerContainerImage), + ApacheContainerImageURL: util.GetEnvVar("RELATED_IMAGE_APACHE_IMAGE_URL_DEFAULT", ApacheContainerImage), + // No default for AmphoraImageContainerImageURL } SetupOctaviaDefaults(octaviaDefaults) diff --git a/api/v1beta1/octavia_webhook.go b/api/v1beta1/octavia_webhook.go index de3a7b42..0c05a574 100644 --- a/api/v1beta1/octavia_webhook.go +++ b/api/v1beta1/octavia_webhook.go @@ -30,6 +30,7 @@ type OctaviaDefaults struct { HousekeepingContainerImageURL string HealthManagerContainerImageURL string WorkerContainerImageURL string + ApacheContainerImageURL string } var octaviaDefaults OctaviaDefaults diff --git a/api/v1beta1/octaviaapi_types.go b/api/v1beta1/octaviaapi_types.go index 90c1a5e0..c76cca5c 100644 --- a/api/v1beta1/octaviaapi_types.go +++ b/api/v1beta1/octaviaapi_types.go @@ -29,6 +29,9 @@ const ( // DeploymentHash hash used to detect changes DeploymentHash = "deployment" + + // ImageUploadHash hash + ImageUploadHash = "image-upload" ) // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index eea6dd0e..72744f46 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -524,6 +524,7 @@ func (in *OctaviaSpecBase) DeepCopyInto(out *OctaviaSpecBase) { *out = make([]OctaviaAmphoraFlavor, len(*in)) copy(*out, *in) } + in.Resources.DeepCopyInto(&out.Resources) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaSpecBase. diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index 21f73c93..747ed061 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -68,6 +68,12 @@ spec: - name type: object type: array + amphoraImageContainerImage: + description: Octavia Container Image URL + type: string + apacheContainerImage: + description: Apache Container Image URL + type: string customServiceConfig: default: '# add your customization here' description: CustomServiceConfig - customize the service config using @@ -1078,10 +1084,63 @@ spec: description: RabbitMQ instance name Needed to request a transportURL that is created and used in Octavia type: string + resources: + description: Resources - Compute Resources required by this service + (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: "Claims lists the names of resources, defined in + spec.resourceClaims, that are used by this container. \n This + is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be set + for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims + of the Pod where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object secret: description: Secret containing OpenStack password information for octavia's keystone password; no longer used for database password type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide Octavia services the default SA name + type: string serviceUser: default: octavia description: ServiceUser - service user name @@ -1097,11 +1156,19 @@ spec: description: LoadBalancerSSHPubKey - The name of the ConfigMap containing the pubilc key for connecting to the amphorae via SSH type: string + tenantName: + default: service + description: TenantName - the name of the OpenStack tenant that controls + the Octavia resources + type: string required: + - apacheContainerImage - databaseInstance - octaviaAPI - rabbitMqClusterName - secret + - serviceAccount + - tenantName type: object status: description: OctaviaStatus defines the observed state of Octavia diff --git a/config/default/manager_default_images.yaml b/config/default/manager_default_images.yaml index ed82601a..6836c990 100644 --- a/config/default/manager_default_images.yaml +++ b/config/default/manager_default_images.yaml @@ -19,3 +19,5 @@ spec: value: quay.io/podified-antelope-centos9/openstack-octavia-health-manager:current-podified - name: RELATED_IMAGE_OCTAVIA_WORKER_IMAGE_URL_DEFAULT value: quay.io/podified-antelope-centos9/openstack-octavia-worker:current-podified + - name: RELATED_IMAGE_APACHE_IMAGE_URL_DEFAULT + value: registry.redhat.io/rhel8/httpd-24:latest diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index 4cf265d5..a9f3f5cd 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -17,8 +17,11 @@ limitations under the License. package controllers import ( + "bufio" "context" "fmt" + "net/http" + "strings" "time" "github.com/go-logr/logr" @@ -26,6 +29,7 @@ import ( redisv1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/deployment" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/job" @@ -44,7 +48,9 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + k8s_labels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" @@ -57,6 +63,7 @@ import ( type OctaviaReconciler struct { client.Client Kclient kubernetes.Interface + Log logr.Logger Scheme *runtime.Scheme } @@ -654,6 +661,53 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav return ctrl.Result{}, err } + // Amphora Image deployment + if instance.Spec.AmphoraImageContainerImage != "" { + hash, err := util.ObjectHash(instance.Spec.AmphoraImageContainerImage) + if err != nil { + return ctrl.Result{}, err + } + if hash != instance.Status.Hash[octaviav1.ImageUploadHash] { + Log.Info("Initializing amphora image upload deployment") + depl := deployment.NewDeployment( + octavia.ImageUploadDeployment(instance, serviceLabels, serviceAnnotations), + time.Duration(5)*time.Second, + ) + ctrlResult, err = depl.CreateOrPatch(ctx, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + readyCount := depl.GetDeployment().Status.ReadyReplicas + if readyCount > 0 { + urlMap, err := r.getLocalImageURLs(ctx, helper, instance) + if err != nil { + return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, err + } + + ok, err := octavia.EnsureAmphoraImages(ctx, instance, &r.Log, helper, urlMap) + if err != nil { + return ctrl.Result{}, err + } + if !ok { + // Images are not ready + return ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil + } + Log.Info(fmt.Sprintf("Setting image upload hash - %s", hash)) + instance.Status.Hash[octaviav1.ImageUploadHash] = hash + + Log.Info("Deleting amphora image upload deployment") + depl.Delete(ctx, helper) + } + } + } else { + if instance.Status.Hash[octaviav1.ImageUploadHash] != "" { + Log.Info("Reseting image upload hash") + instance.Status.Hash[octaviav1.ImageUploadHash] = "" + } + } + // create Deployment - end Log.Info("Reconciled Service successfully") @@ -783,6 +837,45 @@ func (r *OctaviaReconciler) ensureDB( // create service DB - end } +func (r *OctaviaReconciler) getLocalImageURLs( + ctx context.Context, + helper *helper.Helper, + instance *octaviav1.Octavia, +) ([]octavia.OctaviaAmphoraImage, error) { + serviceLabels := map[string]string{common.AppSelector: instance.Name + "-deployment"} + podSelectorString := k8s_labels.Set(serviceLabels).String() + pods, err := helper.GetKClient().CoreV1().Pods( + instance.Namespace).List(ctx, metav1.ListOptions{LabelSelector: podSelectorString}) + if err != nil { + return nil, err + } + host := pods.Items[0].Status.HostIP + + // Get the list of images and their hashes + listUrl := fmt.Sprintf("http://%s:%d/octavia-amphora-images.sha256sum", host, 8080) + + resp, err := http.Get(listUrl) + if err != nil { + return nil, err + } + defer resp.Body.Close() + scanner := bufio.NewScanner(resp.Body) + ret := []octavia.OctaviaAmphoraImage{} + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) == 2 { + name, _ := strings.CutSuffix(fields[1], ".qcow2") + ret = append(ret, octavia.OctaviaAmphoraImage{ + Name: name, + URL: fmt.Sprintf("http://%s:%d/%s", host, 8080, fields[1]), + Checksum: fields[0], + }) + } + } + + return ret, nil +} + // generateServiceConfigMaps - create create configmaps which hold scripts and service configuration // TODO add DefaultConfigOverwrite func (r *OctaviaReconciler) generateServiceConfigMaps( @@ -834,6 +927,7 @@ func (r *OctaviaReconciler) generateServiceConfigMaps( ), } templateParameters["ServiceUser"] = instance.Spec.ServiceUser + templateParameters["ApachePort"] = 8080 cms := []util.Template{ // ScriptsConfigMap diff --git a/main.go b/main.go index c9973dc2..58825ee0 100644 --- a/main.go +++ b/main.go @@ -149,6 +149,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Kclient: kclient, + Log: ctrl.Log.WithName("controllers").WithName("Octavia"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Octavia") os.Exit(1) diff --git a/pkg/octavia/client.go b/pkg/octavia/client.go index a0a8f318..82ecad03 100644 --- a/pkg/octavia/client.go +++ b/pkg/octavia/client.go @@ -27,10 +27,11 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/openstack" + octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" ) -// GetAdminServiceClient - +// GetAdminServiceClient - get a client for the "admin" tenant func GetAdminServiceClient( ctx context.Context, h *helper.Helper, @@ -77,6 +78,54 @@ func GetAdminServiceClient( return os, ctrl.Result{}, nil } +// GetServiceClient - Get a client for the "service" tenant +func GetServiceClient( + ctx context.Context, + h *helper.Helper, + octavia *octaviav1.Octavia, + keystoneAPI *keystonev1.KeystoneAPI, +) (*openstack.OpenStack, ctrl.Result, error) { + // get internal endpoint as authurl from keystone instance + authURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointInternal) + if err != nil { + return nil, ctrl.Result{}, err + } + + // get the password of the admin user from Spec.Secret + // using PasswordSelectors.Admin + authPassword, ctrlResult, err := secret.GetDataFromSecret( + ctx, + h, + octavia.Spec.Secret, + time.Duration(10)*time.Second, + octavia.Spec.PasswordSelectors.Service) + if err != nil { + return nil, ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + return nil, ctrlResult, nil + } + + authOpts := openstack.AuthOpts{ + AuthURL: authURL, + Username: octavia.Spec.ServiceUser, + Password: authPassword, + TenantName: octavia.Spec.TenantName, + DomainName: "Default", + Region: keystoneAPI.Spec.Region, + } + + os, err := openstack.NewOpenStack( + h.GetLogger(), + authOpts, + ) + if err != nil { + return nil, ctrl.Result{}, err + } + + return os, ctrl.Result{}, nil +} + // GetProject - func GetProject(openstack *openstack.OpenStack, projectName string) (*projects.Project, error) { allPages, err := projects.List(openstack.GetOSClient(), projects.ListOpts{Name: projectName}).AllPages() @@ -125,3 +174,12 @@ func GetLoadBalancerClient(o *openstack.OpenStack) (*gophercloud.ServiceClient, } return gophercloudopenstack.NewLoadBalancerV2(o.GetOSClient().ProviderClient, endpointOpts) } + +// GetImageClient - +func GetImageClient(o *openstack.OpenStack) (*gophercloud.ServiceClient, error) { + endpointOpts := gophercloud.EndpointOpts{ + Region: o.GetRegion(), + Availability: gophercloud.AvailabilityInternal, + } + return gophercloudopenstack.NewImageServiceV2(o.GetOSClient().ProviderClient, endpointOpts) +} diff --git a/pkg/octavia/image_upload_deployment.go b/pkg/octavia/image_upload_deployment.go new file mode 100644 index 00000000..b0886ee3 --- /dev/null +++ b/pkg/octavia/image_upload_deployment.go @@ -0,0 +1,174 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package octavia + +import ( + "fmt" + + octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ImageUploadDetails struct { + ContainerImage string + VolumeMounts []corev1.VolumeMount +} + +const ( + // ServiceCommand - + ServiceCommand = "cp -f /usr/local/apache2/conf/httpd.conf /etc/httpd/conf/httpd.conf && /usr/bin/run-httpd" +) + +func getVolumes(name string) []corev1.Volume { + return []corev1.Volume{ + { + Name: "amphora-image", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "httpd-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-config-data", name), + }, + }, + }, + }, + } +} + +func getInitVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "amphora-image", + MountPath: "/usr/local/apache2/htdocs", + }, + } +} + +// GetVolumeMounts - general VolumeMounts +func getVolumeMounts(serviceName string) []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "amphora-image", + MountPath: "/usr/local/apache2/htdocs", + }, + { + Name: "httpd-config", + MountPath: "/usr/local/apache2/conf/httpd.conf", + SubPath: "httpd.conf", + ReadOnly: true, + }, + } +} + +// Deployment func +func ImageUploadDeployment( + instance *octaviav1.Octavia, + labels map[string]string, + annotations map[string]string, +) *appsv1.Deployment { + initVolumeMounts := getInitVolumeMounts() + + // TODO(gthiemonge) healthchecks? + args := []string{"-c", ServiceCommand} + + serviceName := fmt.Sprintf("%s-image-upload", ServiceName) + + depl := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: instance.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.RbacResourceName(), + // TODO(gthiemonge) Expose service on Pod + HostNetwork: true, + Containers: []corev1.Container{ + { + Name: "octavia-amphora-httpd", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ApacheContainerImage, + VolumeMounts: getVolumeMounts("octavia-image-upload"), + Resources: instance.Spec.Resources, + // TODO(gthiemonge) Probes? + }, + //{ + // Name: serviceName, + // Command: []string{ + // "/bin/bash", + // }, + // Image: instance.Spec.AmphoraImageContainerImage, + // SecurityContext: &corev1.SecurityContext{ + // RunAsUser: &runAsUser, + // }, + // VolumeMounts: getVolumeMounts("octavia-image-upload"), + //}, + }, + Volumes: getVolumes(instance.Name), + }, + }, + }, + } + + initContainerDetails := ImageUploadDetails{ + ContainerImage: instance.Spec.AmphoraImageContainerImage, + VolumeMounts: initVolumeMounts, + } + depl.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails) + + return depl +} + +func initContainer(init ImageUploadDetails) []corev1.Container { + runAsUser := int64(0) + envs := []corev1.EnvVar{ + { + Name: "DEST_DIR", + Value: "/usr/local/apache2/htdocs", + }, + } + + return []corev1.Container{ + { + Name: "init", + Image: init.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: envs, + VolumeMounts: getInitVolumeMounts(), + }, + } +} diff --git a/pkg/octavia/images.go b/pkg/octavia/images.go new file mode 100644 index 00000000..3be46668 --- /dev/null +++ b/pkg/octavia/images.go @@ -0,0 +1,237 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package octavia + +import ( + "context" + "fmt" + "strings" + + "github.com/go-logr/logr" + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/imageservice/v2/imageimport" + "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/openstack" + octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" +) + +const ( + AmphoraImageTag = "amphora-image" + AmphoraImageVertTag = "amphora-image-vert" +) + +type OctaviaAmphoraImage struct { + ID string + URL string + Checksum string + Name string + Status images.ImageStatus +} + +// TODO(gthiemonge) Remove when all the clients are used in the octavia controller +func getOpenstackClient( + ctx context.Context, + instance *octaviav1.Octavia, + h *helper.Helper, +) (*openstack.OpenStack, error) { + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) + if err != nil { + return nil, err + } + o, _, err := GetServiceClient(ctx, h, instance, keystoneAPI) + if err != nil { + return nil, err + } + return o, nil +} + +func getTags( + imageName string, +) []string { + if strings.HasPrefix(imageName, "octavia-amphora-vert-") { + return []string{AmphoraImageVertTag} + } else if strings.HasPrefix(imageName, "octavia-amphora-") { + return []string{AmphoraImageTag} + } + return []string{} +} + +func ensureAmphoraImage( + imageClient *gophercloud.ServiceClient, + log *logr.Logger, + amphoraImage OctaviaAmphoraImage, + imageStatus string, +) (bool, error) { + log.Info(fmt.Sprintf("Ensuring image %+v", amphoraImage.Name)) + + // Status is none, the image doesn't exist + if imageStatus == "none" { + visibility := images.ImageVisibilityPrivate + imageCreateOpts := images.CreateOpts{ + Name: amphoraImage.Name, + Visibility: &visibility, + Tags: getTags(amphoraImage.Name), + ContainerFormat: "bare", + DiskFormat: "qcow2", + Properties: map[string]string{ + "hw_architecture": "x86_64", + "image_checksum": amphoraImage.Checksum, + }, + } + + log.Info(fmt.Sprintf("Creating image %s", amphoraImage.Name)) + image, err := images.Create(imageClient, imageCreateOpts).Extract() + if err != nil { + return false, err + } + + imageStatus = string(image.Status) + amphoraImage.ID = image.ID + } + + // Status is queue, image needs to be imported + if imageStatus == string(images.ImageStatusQueued) { + imageImportCreateOpts := imageimport.CreateOpts{ + Name: imageimport.WebDownloadMethod, + URI: amphoraImage.URL, + } + + log.Info(fmt.Sprintf("Uploading image %s", amphoraImage.Name)) + err := imageimport.Create(imageClient, amphoraImage.ID, imageImportCreateOpts).ExtractErr() + if err != nil { + return false, err + } + } + + // Image is active, it's imported and ready + if imageStatus == string(images.ImageStatusActive) { + return true, nil + } + + return false, nil +} + +func amphoraImageListByTag( + imageClient *gophercloud.ServiceClient, + log *logr.Logger, + tag string, +) (map[string]OctaviaAmphoraImage, error) { + listOpts := images.ListOpts{ + Sort: "created_at:desc", + Tags: []string{tag}, + } + + allPages, err := images.List(imageClient, listOpts).AllPages() + if err != nil { + return nil, err + } + + allImages, err := images.ExtractImages(allPages) + if err != nil { + return nil, err + } + + existingAmphoraImages := map[string]OctaviaAmphoraImage{} + for _, image := range allImages { + var checksum string + prop := image.Properties["image_checksum"] + if str, ok := prop.(string); ok { + checksum = str + } else { + checksum = "" + } + if _, ok := existingAmphoraImages[image.Name]; ok { + log.Info(fmt.Sprintf("Multiple '%s' images exist. The Octavia service uses only the most recent image", image.Name)) + } else { + existingAmphoraImages[image.Name] = OctaviaAmphoraImage{ + ID: image.ID, + Name: image.Name, + Status: image.Status, + Checksum: checksum, + } + } + } + + return existingAmphoraImages, nil +} + +func amphoraImageList( + imageClient *gophercloud.ServiceClient, + log *logr.Logger, +) (map[string]OctaviaAmphoraImage, error) { + amphoraImages, err := amphoraImageListByTag(imageClient, log, AmphoraImageTag) + if err != nil { + return nil, err + } + amphoraVertImages, err := amphoraImageListByTag(imageClient, log, AmphoraImageVertTag) + if err != nil { + return nil, err + } + for k, v := range amphoraVertImages { + if _, ok := amphoraImages[k]; !ok { + amphoraImages[k] = v + } + } + return amphoraImages, nil +} + +func EnsureAmphoraImages( + ctx context.Context, + instance *octaviav1.Octavia, + log *logr.Logger, + helper *helper.Helper, + imageList []OctaviaAmphoraImage, +) (bool, error) { + osclient, err := getOpenstackClient(ctx, instance, helper) + if err != nil { + return false, fmt.Errorf("error while getting a service client when creating images: %w", err) + } + + imageClient, err := GetImageClient(osclient) + if err != nil { + return false, fmt.Errorf("error while getting an image client: %w", err) + } + + existingAmphoraImages, err := amphoraImageList(imageClient, log) + if err != nil { + return false, fmt.Errorf("error while getting the list of images: %w", err) + } + + imagesReady := true + for _, amphoraImage := range imageList { + var existingImageStatus string + if existingImage, ok := existingAmphoraImages[amphoraImage.Name]; ok { + existingImageStatus = string(existingImage.Status) + } else { + existingImageStatus = "none" + } + ready, err := ensureAmphoraImage(imageClient, log, amphoraImage, existingImageStatus) + if err != nil { + return false, err + } + if !ready { + imagesReady = false + } + } + + if !imagesReady { + // One of the images is not ready + return false, nil + } + + return true, nil +} diff --git a/templates/octavia/config/httpd.conf b/templates/octavia/config/httpd.conf new file mode 100644 index 00000000..ad2a7c54 --- /dev/null +++ b/templates/octavia/config/httpd.conf @@ -0,0 +1,346 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so 'log/access_log' +# with ServerRoot set to '/www' will be interpreted by the +# server as '/www/log/access_log', where as '/log/access_log' will be +# interpreted as '/log/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/etc/httpd" + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen [::]:{{.ApachePort}} + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +Include conf.modules.d/*.conf + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User default +Group root + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin root@localhost + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +#ServerName www.example.com:80 + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/usr/local/apache2/htdocs" + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options +Indexes +FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # AllowOverride FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog |/usr/bin/cat + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + #CustomLog "logs/access_log" common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + CustomLog |/usr/bin/cat combined + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/opt/rh/httpd24/root/var/www/cgi-bin/" + + + +# +# "/opt/rh/httpd24/root/var/www/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig /etc/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + AddType text/html .shtml + AddOutputFilter INCLUDES .shtml + + +# +# Specify a default charset for all content served; this enables +# interpretation of all content as UTF-8 by default. To use the +# default browser choice (ISO-8859-1), or to allow the META tags +# in HTML content to override this choice, comment out this +# directive: +# +AddDefaultCharset UTF-8 + + + # + # The mod_mime_magic module allows the server to use various hints from the + # contents of the file itself to determine its type. The MIMEMagicFile + # directive tells the module where the hint definitions are located. + # + MIMEMagicFile conf/magic + + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults if commented: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +EnableSendfile on + +# Supplemental configuration +# +# Load config files in the "/etc/httpd/conf.d" directory, except SSL +IncludeOptional conf.d/auth_mellon.conf +IncludeOptional conf.d/autoindex.conf +IncludeOptional conf.d/mod_security.conf +IncludeOptional conf.d/userdir.conf +#IncludeOptional conf.d/welcome.conf diff --git a/templates/octavia/config/mime.types b/templates/octavia/config/mime.types new file mode 100644 index 00000000..1bdf2114 --- /dev/null +++ b/templates/octavia/config/mime.types @@ -0,0 +1,1828 @@ +# This is a comment. I love comments. -*- indent-tabs-mode: t -*- + +# This file controls what Internet media types are sent to the client for +# given file extension(s). Sending the correct media type to the client +# is important so they know how to handle the content of the file. +# Extra types can either be added here or by using an AddType directive +# in your config files. For more information about Internet media types, +# please read RFC 2045, 2046, 2047, 2048, and 2077. The Internet media type +# registry is at . + +# IANA types + +# MIME type Extensions +application/1d-interleaved-parityfec +application/3gpdash-qoe-report+xml +application/3gpp-ims+xml +application/A2L a2l +application/activemessage +application/alto-costmap+json +application/alto-costmapfilter+json +application/alto-directory+json +application/alto-endpointcost+json +application/alto-endpointcostparams+json +application/alto-endpointprop+json +application/alto-endpointpropparams+json +application/alto-error+json +application/alto-networkmap+json +application/alto-networkmapfilter+json +application/AML aml +application/andrew-inset ez +application/applefile +application/ATF atf +application/ATFX atfx +application/ATXML atxml +application/atom+xml atom +application/atomcat+xml atomcat +application/atomdeleted+xml atomdeleted +application/atomicmail +application/atomsvc+xml atomsvc +application/auth-policy+xml apxml +application/bacnet-xdd+zip xdd +application/batch-SMTP +application/beep+xml +application/calendar+json +application/calendar+xml xcs +application/call-completion +application/cals-1840 +application/cbor cbor +application/ccmp+xml ccmp +application/ccxml+xml ccxml +application/CDFX+XML cdfx +application/cdmi-capability cdmia +application/cdmi-container cdmic +application/cdmi-domain cdmid +application/cdmi-object cdmio +application/cdmi-queue cdmiq +application/cdni +application/CEA cea +application/cea-2018+xml +application/cellml+xml cellml cml +application/cfw +application/clue_info+xml clue +application/cms cmsc +application/cnrp+xml +application/coap-group+json +application/coap-payload +application/commonground +application/conference-info+xml +application/cpl+xml cpl +application/cose +application/cose-key +application/cose-key-set +application/csrattrs csrattrs +application/csta+xml +application/CSTAdata+xml +application/csvm+json +application/cybercash +application/dash+xml mpd +application/dashdelta mpdd +application/davmount+xml davmount +application/dca-rft +application/DCD dcd +application/dec-dx +application/dialog-info+xml +application/dicom dcm +application/dicom+json +application/dicom+xml +application/DII dii +application/DIT dit +application/dns +application/dskpp+xml xmls +application/dssc+der dssc +application/dssc+xml xdssc +application/dvcs dvc +application/ecmascript es +application/EDI-Consent +application/EDI-X12 +application/EDIFACT +application/efi efi +application/EmergencyCallData.Comment+xml +application/EmergencyCallData.Control+xml +application/EmergencyCallData.DeviceInfo+xml +application/EmergencyCallData.eCall.MSD +application/EmergencyCallData.ProviderInfo+xml +application/EmergencyCallData.ServiceInfo+xml +application/EmergencyCallData.SubscriberInfo+xml +application/EmergencyCallData.VEDS+xml +application/emma+xml emma +application/emotionml+xml emotionml +application/encaprtp +application/epp+xml +application/epub+zip epub +application/eshop +application/exi exi +application/fastinfoset finf +application/fastsoap +application/fdt+xml fdt +# fits, fit, fts: image/fits +application/fits +# application/font-sfnt deprecated in favor of font/sfnt +application/font-tdpfr pfr +# application/font-woff deprecated in favor of font/woff +application/framework-attributes+xml +application/geo+json geojson +application/geo+json-seq +application/gml+xml gml +application/gzip gz tgz +application/H224 +application/held+xml +application/http +application/hyperstudio stk +application/ibe-key-request+xml +application/ibe-pkg-reply+xml +application/ibe-pp-data +application/iges +application/im-iscomposing+xml +application/index +application/index.cmd +application/index.obj +application/index.response +application/index.vnd +application/inkml+xml ink inkml +application/iotp +application/ipfix ipfix +application/ipp +application/isup +application/its+xml its +application/javascript js +application/jose +application/jose+json +application/jrd+json jrd +application/json json +application/json-patch+json json-patch +application/json-seq +application/jwk+json +application/jwk-set+json +application/jwt +application/kpml-request+xml +application/kpml-response+xml +application/ld+json jsonld +application/lgr+xml lgr +application/link-format wlnk +application/load-control+xml +application/lost+xml lostxml +application/lostsync+xml lostsyncxml +application/LXF lxf +application/mac-binhex40 hqx +application/macwriteii +application/mads+xml mads +application/marc mrc +application/marcxml+xml mrcx +application/mathematica nb ma mb +application/mathml-content+xml +application/mathml-presentation+xml +application/mathml+xml mml +application/mbms-associated-procedure-description+xml +application/mbms-deregister+xml +application/mbms-envelope+xml +application/mbms-msk-response+xml +application/mbms-msk+xml +application/mbms-protection-description+xml +application/mbms-reception-report+xml +application/mbms-register-response+xml +application/mbms-register+xml +application/mbms-schedule+xml +application/mbms-user-service-description+xml +application/mbox mbox +application/media_control+xml +# mpf: text/vnd.ms-mediapackage +application/media-policy-dataset+xml +application/mediaservercontrol+xml +application/merge-patch+json +application/metalink4+xml meta4 +application/mets+xml mets +application/MF4 mf4 +application/mikey +application/mods+xml mods +application/moss-keys +application/moss-signature +application/mosskey-data +application/mosskey-request +application/mp21 m21 mp21 +# mp4, mpg4: video/mp4, see RFC 4337 +application/mp4 +application/mpeg4-generic +application/mpeg4-iod +application/mpeg4-iod-xmt +# xdf: application/xcap-diff+xml +application/mrb-consumer+xml +application/mrb-publish+xml +application/msc-ivr+xml +application/msc-mixer+xml +application/msword doc +application/mud+json +application/mxf mxf +application/n-quads nq +application/n-triples nt +application/nasdata +application/news-checkgroups +application/news-groupinfo +application/news-transmission +application/nlsml+xml +application/nss +application/ocsp-request orq +application/ocsp-response ors +application/octet-stream bin lha lzh exe class so dll img iso +application/oda oda +application/ODX odx +application/oebps-package+xml opf +application/ogg ogx +application/oxps oxps +application/p2p-overlay+xml relo +application/parityfec +# xer: application/xcap-error+xml +application/patch-ops-error+xml +application/pdf pdf +application/PDX pdx +application/pgp-encrypted pgp +application/pgp-keys +application/pgp-signature sig +application/pidf-diff+xml +application/pidf+xml +application/pkcs10 p10 +application/pkcs12 p12 pfx +application/pkcs7-mime p7m p7c +application/pkcs7-signature p7s +application/pkcs8 p8 +# ac: application/vnd.nokia.n-gage.ac+xml +application/pkix-attr-cert +application/pkix-cert cer +application/pkix-crl crl +application/pkix-pkipath pkipath +application/pkixcmp pki +application/pls+xml pls +application/poc-settings+xml +application/postscript ps eps ai +application/ppsp-tracker+json +application/problem+json +application/problem+xml +application/provenance+xml provx +application/prs.alvestrand.titrax-sheet +application/prs.cww cw cww +application/prs.hpub+zip hpub +application/prs.nprend rnd rct +application/prs.plucker +application/prs.rdf-xml-crypt rdf-crypt +application/prs.xsf+xml xsf +application/pskc+xml pskcxml +application/qsig +application/raptorfec +application/rdap+json +application/rdf+xml rdf +application/reginfo+xml rif +application/relax-ng-compact-syntax rnc +application/remote-printing +application/reputon+json +application/resource-lists-diff+xml rld +application/resource-lists+xml rl +application/rfc+xml rfcxml +application/riscos +application/rlmi+xml +application/rls-services+xml rs +application/rpki-ghostbusters gbr +application/rpki-manifest mft +application/rpki-publication +application/rpki-roa roa +application/rpki-updown +application/rtf rtf +application/rtploopback +application/rtx +application/samlassertion+xml +application/samlmetadata+xml +application/sbml+xml +application/scaip+xml +# scm: application/vnd.lotus-screencam +application/scim+json scim +application/scvp-cv-request scq +application/scvp-cv-response scs +application/scvp-vp-request spq +application/scvp-vp-response spp +application/sdp sdp +application/sep+xml +application/sep-exi +application/session-info +application/set-payment +application/set-payment-initiation +application/set-registration +application/set-registration-initiation +application/sgml +application/sgml-open-catalog soc +application/shf+xml shf +application/sieve siv sieve +application/simple-filter+xml cl +application/simple-message-summary +application/simpleSymbolContainer +application/slate +# application/smil obsoleted by application/smil+xml +application/smil+xml smil smi sml +application/smpte336m +application/soap+fastinfoset +application/soap+xml +application/sparql-query rq +application/sparql-results+xml srx +application/spirits-event+xml +application/sql sql +application/srgs gram +application/srgs+xml grxml +application/sru+xml sru +application/ssml+xml ssml +application/tamp-apex-update tau +application/tamp-apex-update-confirm auc +application/tamp-community-update tcu +application/tamp-community-update-confirm cuc +application/tamp-error ter +application/tamp-sequence-adjust tsa +application/tamp-sequence-adjust-confirm sac +# tsq: application/timestamp-query +application/tamp-status-query +# tsr: application/timestamp-reply +application/tamp-status-response +application/tamp-update tur +application/tamp-update-confirm tuc +application/tei+xml tei teiCorpus odd +application/thraud+xml tfi +application/timestamp-query tsq +application/timestamp-reply tsr +application/timestamped-data tsd +application/trig trig +application/ttml+xml ttml +application/tve-trigger +application/ulpfec +application/urc-grpsheet+xml gsheet +application/urc-ressheet+xml rsheet +application/urc-targetdesc+xml td +application/urc-uisocketdesc+xml uis +application/vcard+json +application/vcard+xml +application/vemmi +application/vnd.3gpp.access-transfer-events+xml +application/vnd.3gpp.bsf+xml +application/vnd.3gpp.mid-call+xml +application/vnd.3gpp.pic-bw-large plb +application/vnd.3gpp.pic-bw-small psb +application/vnd.3gpp.pic-bw-var pvb +application/vnd.3gpp-prose+xml +application/vnd.3gpp-prose-pc3ch+xml +# sms: application/vnd.3gpp2.sms +application/vnd.3gpp.sms +application/vnd.3gpp.sms+xml +application/vnd.3gpp.srvcc-ext+xml +application/vnd.3gpp.SRVCC-info+xml +application/vnd.3gpp.state-and-event-info+xml +application/vnd.3gpp.ussd+xml +application/vnd.3gpp2.bcmcsinfo+xml +application/vnd.3gpp2.sms sms +application/vnd.3gpp2.tcap tcap +application/vnd.3lightssoftware.imagescal imgcal +application/vnd.3M.Post-it-Notes pwn +application/vnd.accpac.simply.aso aso +application/vnd.accpac.simply.imp imp +application/vnd.acucobol acu +application/vnd.acucorp atc acutc +application/vnd.adobe.flash.movie swf +application/vnd.adobe.formscentral.fcdt fcdt +application/vnd.adobe.fxp fxp fxpl +application/vnd.adobe.partial-upload +application/vnd.adobe.xdp+xml xdp +application/vnd.adobe.xfdf xfdf +application/vnd.aether.imp +application/vnd.ah-barcode +application/vnd.ahead.space ahead +application/vnd.airzip.filesecure.azf azf +application/vnd.airzip.filesecure.azs azs +application/vnd.amazon.mobi8-ebook azw3 +application/vnd.americandynamics.acc acc +application/vnd.amiga.ami ami +application/vnd.amundsen.maze+xml +application/vnd.anki apkg +application/vnd.anser-web-certificate-issue-initiation cii +# Not in IANA listing, but is on FTP site? +application/vnd.anser-web-funds-transfer-initiation fti +# atx: audio/ATRAC-X +application/vnd.antix.game-component +application/vnd.apache.thrift.binary +application/vnd.apache.thrift.compact +application/vnd.apache.thrift.json +application/vnd.api+json +application/vnd.apothekende.reservation+json +application/vnd.apple.installer+xml dist distz pkg mpkg +# m3u: audio/x-mpegurl for now +application/vnd.apple.mpegurl m3u8 +# application/vnd.arastra.swi obsoleted by application/vnd.aristanetworks.swi +application/vnd.aristanetworks.swi swi +application/vnd.artsquare +application/vnd.astraea-software.iota iota +application/vnd.audiograph aep +application/vnd.autopackage package +application/vnd.avistar+xml +application/vnd.balsamiq.bmml+xml bmml +application/vnd.balsamiq.bmpr bmpr +application/vnd.bekitzur-stech+json +application/vnd.bint.med-content +application/vnd.biopax.rdf+xml +application/vnd.blueice.multipass mpm +application/vnd.bluetooth.ep.oob ep +application/vnd.bluetooth.le.oob le +application/vnd.bmi bmi +application/vnd.businessobjects rep +application/vnd.cab-jscript +application/vnd.canon-cpdl +application/vnd.canon-lips +application/vnd.capasystems-pg+json +application/vnd.cendio.thinlinc.clientconf tlclient +application/vnd.century-systems.tcp_stream +application/vnd.chemdraw+xml cdxml +application/vnd.chess-pgn pgn +application/vnd.chipnuts.karaoke-mmd mmd +application/vnd.cinderella cdy +application/vnd.cirpack.isdn-ext +application/vnd.citationstyles.style+xml csl +application/vnd.claymore cla +application/vnd.cloanto.rp9 rp9 +application/vnd.clonk.c4group c4g c4d c4f c4p c4u +application/vnd.cluetrust.cartomobile-config c11amc +application/vnd.cluetrust.cartomobile-config-pkg c11amz +application/vnd.coffeescript coffee +application/vnd.collection+json +application/vnd.collection.doc+json +application/vnd.collection.next+json +application/vnd.comicbook+zip cbz +# icc: application/vnd.iccprofile +application/vnd.commerce-battelle ica icf icd ic0 ic1 ic2 ic3 ic4 ic5 ic6 ic7 ic8 +application/vnd.commonspace csp cst +application/vnd.contact.cmsg cdbcmsg +application/vnd.coreos.ignition+json ign ignition +application/vnd.cosmocaller cmc +application/vnd.crick.clicker clkx +application/vnd.crick.clicker.keyboard clkk +application/vnd.crick.clicker.palette clkp +application/vnd.crick.clicker.template clkt +application/vnd.crick.clicker.wordbank clkw +application/vnd.criticaltools.wbs+xml wbs +application/vnd.ctc-posml pml +application/vnd.ctct.ws+xml +application/vnd.cups-pdf +application/vnd.cups-postscript +application/vnd.cups-ppd ppd +application/vnd.cups-raster +application/vnd.cups-raw +application/vnd.curl curl +application/vnd.cyan.dean.root+xml +application/vnd.cybank +application/vnd.d2l.coursepackage1p0+zip +application/vnd.dart dart +application/vnd.data-vision.rdz rdz +application/vnd.datapackage+json +application/vnd.dataresource+json +application/vnd.debian.binary-package deb udeb +application/vnd.dece.data uvf uvvf uvd uvvd +application/vnd.dece.ttml+xml uvt uvvt +application/vnd.dece.unspecified uvx uvvx +application/vnd.dece.zip uvz uvvz +application/vnd.denovo.fcselayout-link fe_launch +application/vnd.desmume.movie dsm +application/vnd.dir-bi.plate-dl-nosuffix +application/vnd.dm.delegation+xml +application/vnd.dna dna +application/vnd.document+json docjson +application/vnd.dolby.mobile.1 +application/vnd.dolby.mobile.2 +application/vnd.doremir.scorecloud-binary-document scld +application/vnd.dpgraph dpg mwc dpgraph +application/vnd.dreamfactory dfac +application/vnd.drive+json +application/vnd.dtg.local +application/vnd.dtg.local.flash fla +application/vnd.dtg.local.html +application/vnd.dvb.ait ait +# class: application/octet-stream +application/vnd.dvb.dvbj +application/vnd.dvb.esgcontainer +application/vnd.dvb.ipdcdftnotifaccess +application/vnd.dvb.ipdcesgaccess +application/vnd.dvb.ipdcesgaccess2 +application/vnd.dvb.ipdcesgpdd +application/vnd.dvb.ipdcroaming +application/vnd.dvb.iptv.alfec-base +application/vnd.dvb.iptv.alfec-enhancement +application/vnd.dvb.notif-aggregate-root+xml +application/vnd.dvb.notif-container+xml +application/vnd.dvb.notif-generic+xml +application/vnd.dvb.notif-ia-msglist+xml +application/vnd.dvb.notif-ia-registration-request+xml +application/vnd.dvb.notif-ia-registration-response+xml +application/vnd.dvb.notif-init+xml +# pfr: application/font-tdpfr +application/vnd.dvb.pfr +application/vnd.dvb.service svc +# dxr: application/x-director +application/vnd.dxr +application/vnd.dynageo geo +application/vnd.dzr dzr +application/vnd.easykaraoke.cdgdownload +application/vnd.ecdis-update +application/vnd.ecowin.chart mag +application/vnd.ecowin.filerequest +application/vnd.ecowin.fileupdate +application/vnd.ecowin.series +application/vnd.ecowin.seriesrequest +application/vnd.ecowin.seriesupdate +# img: application/octet-stream +application/vnd.efi-img +# iso: application/octet-stream +application/vnd.efi-iso +application/vnd.enliven nml +application/vnd.enphase.envoy +application/vnd.eprints.data+xml +application/vnd.epson.esf esf +application/vnd.epson.msf msf +application/vnd.epson.quickanime qam +application/vnd.epson.salt slt +application/vnd.epson.ssf ssf +application/vnd.ericsson.quickcall qcall qca +application/vnd.espass-espass+zip espass +application/vnd.eszigno3+xml es3 et3 +application/vnd.etsi.aoc+xml +application/vnd.etsi.asic-e+zip asice sce +# scs: application/scvp-cv-response +application/vnd.etsi.asic-s+zip asics +application/vnd.etsi.cug+xml +application/vnd.etsi.iptvcommand+xml +application/vnd.etsi.iptvdiscovery+xml +application/vnd.etsi.iptvprofile+xml +application/vnd.etsi.iptvsad-bc+xml +application/vnd.etsi.iptvsad-cod+xml +application/vnd.etsi.iptvsad-npvr+xml +application/vnd.etsi.iptvservice+xml +application/vnd.etsi.iptvsync+xml +application/vnd.etsi.iptvueprofile+xml +application/vnd.etsi.mcid+xml +application/vnd.etsi.mheg5 +application/vnd.etsi.overload-control-policy-dataset+xml +application/vnd.etsi.pstn+xml +application/vnd.etsi.sci+xml +application/vnd.etsi.simservs+xml +application/vnd.etsi.timestamp-token tst +application/vnd.etsi.tsl.der +application/vnd.etsi.tsl+xml +application/vnd.eudora.data +application/vnd.ezpix-album ez2 +application/vnd.ezpix-package ez3 +application/vnd.f-secure.mobile +application/vnd.fastcopy-disk-image dim +application/vnd.fdf fdf +application/vnd.fdsn.mseed msd mseed +application/vnd.fdsn.seed seed dataless +application/vnd.ffsns +application/vnd.filmit.zfc zfc +# all extensions: application/vnd.hbci +application/vnd.fints +application/vnd.firemonkeys.cloudcell +application/vnd.FloGraphIt gph +application/vnd.fluxtime.clip ftc +application/vnd.font-fontforge-sfd sfd +application/vnd.framemaker fm +application/vnd.frogans.fnc fnc +application/vnd.frogans.ltf ltf +application/vnd.fsc.weblaunch fsc +application/vnd.fujitsu.oasys oas +application/vnd.fujitsu.oasys2 oa2 +application/vnd.fujitsu.oasys3 oa3 +application/vnd.fujitsu.oasysgp fg5 +application/vnd.fujitsu.oasysprs bh2 +application/vnd.fujixerox.ART-EX +application/vnd.fujixerox.ART4 +application/vnd.fujixerox.ddd ddd +application/vnd.fujixerox.docuworks xdw +application/vnd.fujixerox.docuworks.binder xbd +application/vnd.fujixerox.docuworks.container xct +application/vnd.fujixerox.HBPL +application/vnd.fut-misnet +application/vnd.fuzzysheet fzs +application/vnd.genomatix.tuxedo txd +# application/vnd.geo+json obsoleted by application/geo+json +application/vnd.geocube+xml g3 g³ +application/vnd.geogebra.file ggb +application/vnd.geogebra.tool ggt +application/vnd.geometry-explorer gex gre +application/vnd.geonext gxt +application/vnd.geoplan g2w +application/vnd.geospace g3w +# gbr: application/rpki-ghostbusters +application/vnd.gerber +application/vnd.globalplatform.card-content-mgt +application/vnd.globalplatform.card-content-mgt-response +application/vnd.gmx gmx +application/vnd.google-earth.kml+xml kml +application/vnd.google-earth.kmz kmz +application/vnd.gov.sk.e-form+xml +application/vnd.gov.sk.e-form+zip +application/vnd.gov.sk.xmldatacontainer+xml +application/vnd.grafeq gqf gqs +application/vnd.gridmp +application/vnd.groove-account gac +application/vnd.groove-help ghf +application/vnd.groove-identity-message gim +application/vnd.groove-injector grv +application/vnd.groove-tool-message gtm +application/vnd.groove-tool-template tpl +application/vnd.groove-vcard vcg +application/vnd.hal+json +application/vnd.hal+xml hal +application/vnd.HandHeld-Entertainment+xml zmm +application/vnd.hbci hbci hbc kom upa pkd bpd +application/vnd.hc+json +# rep: application/vnd.businessobjects +application/vnd.hcl-bireports +application/vnd.hdt hdt +application/vnd.heroku+json +application/vnd.hhe.lesson-player les +application/vnd.hp-HPGL hpgl +application/vnd.hp-hpid hpi hpid +application/vnd.hp-hps hps +application/vnd.hp-jlyt jlt +application/vnd.hp-PCL pcl +application/vnd.hp-PCLXL +application/vnd.httphone +application/vnd.hydrostatix.sof-data sfd-hdstx +application/vnd.hyperdrive+json +application/vnd.hzn-3d-crossword x3d +application/vnd.ibm.afplinedata +application/vnd.ibm.electronic-media emm +application/vnd.ibm.MiniPay mpy +application/vnd.ibm.modcap list3820 listafp afp pseg3820 +application/vnd.ibm.rights-management irm +application/vnd.ibm.secure-container sc +application/vnd.iccprofile icc icm +application/vnd.ieee.1905 1905.1 +application/vnd.igloader igl +application/vnd.imagemeter.folder+zip imf +application/vnd.imagemeter.image+zip imi +application/vnd.immervision-ivp ivp +application/vnd.immervision-ivu ivu +application/vnd.ims.imsccv1p1 imscc +application/vnd.ims.imsccv1p2 +application/vnd.ims.imsccv1p3 +application/vnd.ims.lis.v2.result+json +application/vnd.ims.lti.v2.toolconsumerprofile+json +application/vnd.ims.lti.v2.toolproxy.id+json +application/vnd.ims.lti.v2.toolproxy+json +application/vnd.ims.lti.v2.toolsettings+json +application/vnd.ims.lti.v2.toolsettings.simple+json +application/vnd.informedcontrol.rms+xml +# application/vnd.informix-visionary obsoleted by application/vnd.visionary +application/vnd.infotech.project +application/vnd.infotech.project+xml +application/vnd.innopath.wamp.notification +application/vnd.insors.igm igm +application/vnd.intercon.formnet xpw xpx +application/vnd.intergeo i2g +application/vnd.intertrust.digibox +application/vnd.intertrust.nncp +application/vnd.intu.qbo qbo +application/vnd.intu.qfx qfx +application/vnd.iptc.g2.catalogitem+xml +application/vnd.iptc.g2.conceptitem+xml +application/vnd.iptc.g2.knowledgeitem+xml +application/vnd.iptc.g2.newsitem+xml +application/vnd.iptc.g2.newsmessage+xml +application/vnd.iptc.g2.packageitem+xml +application/vnd.iptc.g2.planningitem+xml +application/vnd.ipunplugged.rcprofile rcprofile +application/vnd.irepository.package+xml irp +application/vnd.is-xpr xpr +application/vnd.isac.fcs fcs +application/vnd.jam jam +application/vnd.japannet-directory-service +application/vnd.japannet-jpnstore-wakeup +application/vnd.japannet-payment-wakeup +application/vnd.japannet-registration +application/vnd.japannet-registration-wakeup +application/vnd.japannet-setstore-wakeup +application/vnd.japannet-verification +application/vnd.japannet-verification-wakeup +application/vnd.jcp.javame.midlet-rms rms +application/vnd.jisp jisp +application/vnd.joost.joda-archive joda +application/vnd.jsk.isdn-ngn +application/vnd.kahootz ktz ktr +application/vnd.kde.karbon karbon +application/vnd.kde.kchart chrt +application/vnd.kde.kformula kfo +application/vnd.kde.kivio flw +application/vnd.kde.kontour kon +application/vnd.kde.kpresenter kpr kpt +application/vnd.kde.kspread ksp +application/vnd.kde.kword kwd kwt +application/vnd.kenameaapp htke +application/vnd.kidspiration kia +application/vnd.Kinar kne knp sdf +application/vnd.koan skp skd skm skt +application/vnd.kodak-descriptor sse +application/vnd.las.las+json lasjson +application/vnd.las.las+xml lasxml +application/vnd.liberty-request+xml +application/vnd.llamagraphics.life-balance.desktop lbd +application/vnd.llamagraphics.life-balance.exchange+xml lbe +application/vnd.lotus-1-2-3 123 wk4 wk3 wk1 +application/vnd.lotus-approach apr vew +application/vnd.lotus-freelance prz pre +application/vnd.lotus-notes nsf ntf ndl ns4 ns3 ns2 nsh nsg +application/vnd.lotus-organizer or3 or2 org +application/vnd.lotus-screencam scm +application/vnd.lotus-wordpro lwp sam +application/vnd.macports.portpkg portpkg +application/vnd.mapbox-vector-tile mvt +application/vnd.marlin.drm.actiontoken+xml +application/vnd.marlin.drm.conftoken+xml +application/vnd.marlin.drm.license+xml +application/vnd.marlin.drm.mdcf mdc +application/vnd.mason+json +application/vnd.maxmind.maxmind-db mmdb +application/vnd.mcd mcd +application/vnd.medcalcdata mc1 +application/vnd.mediastation.cdkey cdkey +application/vnd.meridian-slingshot +application/vnd.MFER mwf +application/vnd.mfmp mfm +application/vnd.micro+json +application/vnd.micrografx.flo flo +application/vnd.micrografx.igx igx +application/vnd.microsoft.portable-executable +application/vnd.microsoft.windows.thumbnail-cache +application/vnd.miele+json +application/vnd.mif mif +application/vnd.minisoft-hp3000-save +application/vnd.mitsubishi.misty-guard.trustweb +application/vnd.Mobius.DAF daf +application/vnd.Mobius.DIS dis +application/vnd.Mobius.MBK mbk +application/vnd.Mobius.MQY mqy +application/vnd.Mobius.MSL msl +application/vnd.Mobius.PLC plc +application/vnd.Mobius.TXF txf +application/vnd.mophun.application mpn +application/vnd.mophun.certificate mpc +application/vnd.motorola.flexsuite +application/vnd.motorola.flexsuite.adsi +application/vnd.motorola.flexsuite.fis +application/vnd.motorola.flexsuite.gotap +application/vnd.motorola.flexsuite.kmr +application/vnd.motorola.flexsuite.ttc +application/vnd.motorola.flexsuite.wem +application/vnd.motorola.iprm +application/vnd.mozilla.xul+xml xul +application/vnd.ms-3mfdocument 3mf +application/vnd.ms-artgalry cil +application/vnd.ms-asf asf +application/vnd.ms-cab-compressed cab +application/vnd.ms-excel xls xlm xla xlc xlt xlw +application/vnd.ms-excel.template.macroEnabled.12 xltm +application/vnd.ms-excel.addin.macroEnabled.12 xlam +application/vnd.ms-excel.sheet.binary.macroEnabled.12 xlsb +application/vnd.ms-excel.sheet.macroEnabled.12 xlsm +application/vnd.ms-fontobject eot +application/vnd.ms-htmlhelp chm +application/vnd.ms-ims ims +application/vnd.ms-lrm lrm +application/vnd.ms-office.activeX+xml +application/vnd.ms-officetheme thmx +application/vnd.ms-playready.initiator+xml +application/vnd.ms-powerpoint ppt pps pot +application/vnd.ms-powerpoint.addin.macroEnabled.12 ppam +application/vnd.ms-powerpoint.presentation.macroEnabled.12 pptm +application/vnd.ms-powerpoint.slide.macroEnabled.12 sldm +application/vnd.ms-powerpoint.slideshow.macroEnabled.12 ppsm +application/vnd.ms-powerpoint.template.macroEnabled.12 potm +application/vnd.ms-PrintDeviceCapabilities+xml +application/vnd.ms-PrintSchemaTicket+xml +application/vnd.ms-project mpp mpt +application/vnd.ms-tnef tnef tnf +application/vnd.ms-windows.devicepairing +application/vnd.ms-windows.nwprinting.oob +application/vnd.ms-windows.printerpairing +application/vnd.ms-windows.wsd.oob +application/vnd.ms-wmdrm.lic-chlg-req +application/vnd.ms-wmdrm.lic-resp +application/vnd.ms-wmdrm.meter-chlg-req +application/vnd.ms-wmdrm.meter-resp +application/vnd.ms-word.document.macroEnabled.12 docm +application/vnd.ms-word.template.macroEnabled.12 dotm +application/vnd.ms-works wcm wdb wks wps +application/vnd.ms-wpl wpl +application/vnd.ms-xpsdocument xps +application/vnd.msa-disk-image msa +application/vnd.mseq mseq +application/vnd.msign +application/vnd.multiad.creator crtr +application/vnd.multiad.creator.cif cif +application/vnd.music-niff +application/vnd.musician mus +application/vnd.muvee.style msty +application/vnd.mynfc taglet +application/vnd.ncd.control +application/vnd.ncd.reference +application/vnd.nearst.inv+json +application/vnd.nervana entity request bkm kcm +application/vnd.netfpx +# ntf: application/vnd.lotus-notes +application/vnd.nitf nitf +application/vnd.neurolanguage.nlu nlu +application/vnd.nintendo.nitro.rom nds +application/vnd.nintendo.snes.rom sfc smc +application/vnd.noblenet-directory nnd +application/vnd.noblenet-sealer nns +application/vnd.noblenet-web nnw +application/vnd.nokia.catalogs +application/vnd.nokia.conml+wbxml +application/vnd.nokia.conml+xml +application/vnd.nokia.iptv.config+xml +application/vnd.nokia.iSDS-radio-presets +application/vnd.nokia.landmark+wbxml +application/vnd.nokia.landmark+xml +application/vnd.nokia.landmarkcollection+xml +application/vnd.nokia.n-gage.ac+xml ac +application/vnd.nokia.n-gage.data ngdat +application/vnd.nokia.n-gage.symbian.install n-gage +application/vnd.nokia.ncd +application/vnd.nokia.pcd+wbxml +application/vnd.nokia.pcd+xml +application/vnd.nokia.radio-preset rpst +application/vnd.nokia.radio-presets rpss +application/vnd.novadigm.EDM edm +application/vnd.novadigm.EDX edx +application/vnd.novadigm.EXT ext +application/vnd.ntt-local.content-share +application/vnd.ntt-local.file-transfer +application/vnd.ntt-local.ogw_remote-access +application/vnd.ntt-local.sip-ta_remote +application/vnd.ntt-local.sip-ta_tcp_stream +application/vnd.oasis.opendocument.chart odc +application/vnd.oasis.opendocument.chart-template otc +application/vnd.oasis.opendocument.database odb +application/vnd.oasis.opendocument.formula odf +# otf: font/otf +application/vnd.oasis.opendocument.formula-template +application/vnd.oasis.opendocument.graphics odg +application/vnd.oasis.opendocument.graphics-template otg +application/vnd.oasis.opendocument.image odi +application/vnd.oasis.opendocument.image-template oti +application/vnd.oasis.opendocument.presentation odp +application/vnd.oasis.opendocument.presentation-template otp +application/vnd.oasis.opendocument.spreadsheet ods +application/vnd.oasis.opendocument.spreadsheet-template ots +application/vnd.oasis.opendocument.text odt +application/vnd.oasis.opendocument.text-master odm +application/vnd.oasis.opendocument.text-template ott +application/vnd.oasis.opendocument.text-web oth +application/vnd.obn +application/vnd.ocf+cbor +application/vnd.oftn.l10n+json +application/vnd.oipf.contentaccessdownload+xml +application/vnd.oipf.contentaccessstreaming+xml +application/vnd.oipf.cspg-hexbinary +application/vnd.oipf.dae.svg+xml +application/vnd.oipf.dae.xhtml+xml +application/vnd.oipf.mippvcontrolmessage+xml +application/vnd.oipf.pae.gem +application/vnd.oipf.spdiscovery+xml +application/vnd.oipf.spdlist+xml +application/vnd.oipf.ueprofile+xml +application/vnd.olpc-sugar xo +application/vnd.oma.bcast.associated-procedure-parameter+xml +application/vnd.oma.bcast.drm-trigger+xml +application/vnd.oma.bcast.imd+xml +application/vnd.oma.bcast.ltkm +application/vnd.oma.bcast.notification+xml +application/vnd.oma.bcast.provisioningtrigger +application/vnd.oma.bcast.sgboot +application/vnd.oma.bcast.sgdd+xml +application/vnd.oma.bcast.sgdu +application/vnd.oma.bcast.simple-symbol-container +application/vnd.oma.bcast.smartcard-trigger+xml +application/vnd.oma.bcast.sprov+xml +application/vnd.oma.bcast.stkm +application/vnd.oma.cab-address-book+xml +application/vnd.oma.cab-feature-handler+xml +application/vnd.oma.cab-pcc+xml +application/vnd.oma.cab-subs-invite+xml +application/vnd.oma.cab-user-prefs+xml +application/vnd.oma.dcd +application/vnd.oma.dcdc +application/vnd.oma.dd2+xml dd2 +application/vnd.oma.drm.risd+xml +application/vnd.oma.group-usage-list+xml +application/vnd.oma.lwm2m+json +application/vnd.oma.lwm2m+tlv +application/vnd.oma.pal+xml +application/vnd.oma.poc.detailed-progress-report+xml +application/vnd.oma.poc.final-report+xml +application/vnd.oma.poc.groups+xml +application/vnd.oma.poc.invocation-descriptor+xml +application/vnd.oma.poc.optimized-progress-report+xml +application/vnd.oma.push +application/vnd.oma.scidm.messages+xml +application/vnd.oma.xcap-directory+xml +application/vnd.oma-scws-config +application/vnd.oma-scws-http-request +application/vnd.oma-scws-http-response +application/vnd.omads-email+xml +application/vnd.omads-file+xml +application/vnd.omads-folder+xml +application/vnd.omaloc-supl-init +application/vnd.onepager tam +application/vnd.onepagertamp tamp +application/vnd.onepagertamx tamx +application/vnd.onepagertat tat +application/vnd.onepagertatp tatp +application/vnd.onepagertatx tatx +application/vnd.openblox.game+xml obgx +application/vnd.openblox.game-binary obg +application/vnd.openeye.oeb oeb +application/vnd.openofficeorg.extension oxt +application/vnd.openstreetmap.data+xml osm +application/vnd.openxmlformats-officedocument.custom-properties+xml +application/vnd.openxmlformats-officedocument.customXmlProperties+xml +application/vnd.openxmlformats-officedocument.drawing+xml +application/vnd.openxmlformats-officedocument.drawingml.chart+xml +application/vnd.openxmlformats-officedocument.drawingml.chartshapes+xml +application/vnd.openxmlformats-officedocument.drawingml.diagramColors+xml +application/vnd.openxmlformats-officedocument.drawingml.diagramData+xml +application/vnd.openxmlformats-officedocument.drawingml.diagramLayout+xml +application/vnd.openxmlformats-officedocument.drawingml.diagramStyle+xml +application/vnd.openxmlformats-officedocument.extended-properties+xml +application/vnd.openxmlformats-officedocument.presentationml.commentAuthors+xml +application/vnd.openxmlformats-officedocument.presentationml.comments+xml +application/vnd.openxmlformats-officedocument.presentationml.handoutMaster+xml +application/vnd.openxmlformats-officedocument.presentationml.notesMaster+xml +application/vnd.openxmlformats-officedocument.presentationml.notesSlide+xml +application/vnd.openxmlformats-officedocument.presentationml.presProps+xml +application/vnd.openxmlformats-officedocument.presentationml.presentation pptx +application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml +application/vnd.openxmlformats-officedocument.presentationml.slide sldx +application/vnd.openxmlformats-officedocument.presentationml.slide+xml +application/vnd.openxmlformats-officedocument.presentationml.slideLayout+xml +application/vnd.openxmlformats-officedocument.presentationml.slideMaster+xml +application/vnd.openxmlformats-officedocument.presentationml.slideUpdateInfo+xml +application/vnd.openxmlformats-officedocument.presentationml.slideshow ppsx +application/vnd.openxmlformats-officedocument.presentationml.slideshow.main+xml +application/vnd.openxmlformats-officedocument.presentationml.tableStyles+xml +application/vnd.openxmlformats-officedocument.presentationml.tags+xml +application/vnd.openxmlformats-officedocument.presentationml.template potx +application/vnd.openxmlformats-officedocument.presentationml.template.main+xml +application/vnd.openxmlformats-officedocument.presentationml.viewProps+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.calcChain+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.chartsheet+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.comments+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.connections+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.dialogsheet+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.externalLink+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheDefinition+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheRecords+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.pivotTable+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.queryTable+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.revisionHeaders+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.revisionLog+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx +application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.sheetMetadata+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.table+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.tableSingleCells+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.template xltx +application/vnd.openxmlformats-officedocument.spreadsheetml.template.main+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.userNames+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.volatileDependencies+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml +application/vnd.openxmlformats-officedocument.theme+xml +application/vnd.openxmlformats-officedocument.themeOverride+xml +application/vnd.openxmlformats-officedocument.vmlDrawing +application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.document docx +application/vnd.openxmlformats-officedocument.wordprocessingml.document.glossary+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.endnotes+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.fontTable+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.footnotes+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.template dotx +application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.webSettings+xml +application/vnd.openxmlformats-package.core-properties+xml +application/vnd.openxmlformats-package.digital-signature-xmlsignature+xml +application/vnd.openxmlformats-package.relationships+xml +application/vnd.oracle.resource+json +application/vnd.orange.indata +application/vnd.osa.netdeploy ndc +application/vnd.osgeo.mapguide.package mgp +# jar: application/x-java-archive +application/vnd.osgi.bundle +application/vnd.osgi.dp dp +application/vnd.osgi.subsystem esa +application/vnd.otps.ct-kip+xml +application/vnd.oxli.countgraph oxlicg +application/vnd.pagerduty+json +application/vnd.palm prc pdb pqa oprc +application/vnd.panoply plp +application/vnd.paos+xml +application/vnd.pawaafile paw +application/vnd.pcos +application/vnd.pg.format str +application/vnd.pg.osasli ei6 +application/vnd.piaccess.application-license pil +application/vnd.picsel efif +application/vnd.pmi.widget wg +application/vnd.poc.group-advertisement+xml +application/vnd.pocketlearn plf +application/vnd.powerbuilder6 pbd +application/vnd.powerbuilder6-s +application/vnd.powerbuilder7 +application/vnd.powerbuilder7-s +application/vnd.powerbuilder75 +application/vnd.powerbuilder75-s +application/vnd.preminet preminet +application/vnd.previewsystems.box box vbox +application/vnd.proteus.magazine mgz +application/vnd.publishare-delta-tree qps +# pti: image/prs.pti +application/vnd.pvi.ptid1 ptid +application/vnd.pwg-multiplexed +application/vnd.pwg-xhtml-print+xml +application/vnd.qualcomm.brew-app-res bar +application/vnd.quarantainenet +application/vnd.Quark.QuarkXPress qxd qxt qwd qwt qxl qxb +application/vnd.quobject-quoxdocument quox quiz +application/vnd.radisys.moml+xml +application/vnd.radisys.msml-audit-conf+xml +application/vnd.radisys.msml-audit-conn+xml +application/vnd.radisys.msml-audit-dialog+xml +application/vnd.radisys.msml-audit-stream+xml +application/vnd.radisys.msml-audit+xml +application/vnd.radisys.msml-conf+xml +application/vnd.radisys.msml-dialog-base+xml +application/vnd.radisys.msml-dialog-fax-detect+xml +application/vnd.radisys.msml-dialog-fax-sendrecv+xml +application/vnd.radisys.msml-dialog-group+xml +application/vnd.radisys.msml-dialog-speech+xml +application/vnd.radisys.msml-dialog-transform+xml +application/vnd.radisys.msml-dialog+xml +application/vnd.radisys.msml+xml +application/vnd.rainstor.data tree +application/vnd.rapid +application/vnd.rar rar +application/vnd.realvnc.bed bed +application/vnd.recordare.musicxml mxl +application/vnd.recordare.musicxml+xml +application/vnd.RenLearn.rlprint +application/vnd.rig.cryptonote cryptonote +application/vnd.route66.link66+xml link66 +# gbr: application/rpki-ghostbusters +application/vnd.rs-274x +application/vnd.ruckus.download +application/vnd.s3sms +application/vnd.sailingtracker.track st +application/vnd.sbm.cid +application/vnd.sbm.mid2 +application/vnd.scribus scd sla slaz +application/vnd.sealed.3df s3df +application/vnd.sealed.csf scsf +application/vnd.sealed.doc sdoc sdo s1w +application/vnd.sealed.eml seml sem +application/vnd.sealed.mht smht smh +application/vnd.sealed.net +# spp: application/scvp-vp-response +application/vnd.sealed.ppt sppt s1p +application/vnd.sealed.tiff stif +application/vnd.sealed.xls sxls sxl s1e +# stm: audio/x-stm +application/vnd.sealedmedia.softseal.html stml s1h +application/vnd.sealedmedia.softseal.pdf spdf spd s1a +application/vnd.seemail see +application/vnd.sema sema +application/vnd.semd semd +application/vnd.semf semf +application/vnd.shana.informed.formdata ifm +application/vnd.shana.informed.formtemplate itp +application/vnd.shana.informed.interchange iif +application/vnd.shana.informed.package ipk +application/vnd.SimTech-MindMapper twd twds +application/vnd.siren+json +application/vnd.smaf mmf +application/vnd.smart.notebook notebook +application/vnd.smart.teacher teacher +application/vnd.software602.filler.form+xml fo +application/vnd.software602.filler.form-xml-zip zfo +application/vnd.solent.sdkm+xml sdkm sdkd +application/vnd.spotfire.dxp dxp +application/vnd.spotfire.sfs sfs +application/vnd.sss-cod +application/vnd.sss-dtf +application/vnd.sss-ntf +application/vnd.stepmania.package smzip +application/vnd.stepmania.stepchart sm +application/vnd.street-stream +application/vnd.sun.wadl+xml wadl +application/vnd.sus-calendar sus susp +application/vnd.svd +application/vnd.swiftview-ics +application/vnd.syncml+xml xsm +application/vnd.syncml.dm+wbxml bdm +application/vnd.syncml.dm+xml xdm +application/vnd.syncml.dm.notification +application/vnd.syncml.dmddf+wbxml +application/vnd.syncml.dmddf+xml ddf +application/vnd.syncml.dmtnds+wbxml +application/vnd.syncml.dmtnds+xml +application/vnd.syncml.ds.notification +application/vnd.tableschema+json +application/vnd.tao.intent-module-archive tao +application/vnd.tcpdump.pcap pcap cap dmp +application/vnd.theqvd qvd +application/vnd.tmd.mediaflex.api+xml +application/vnd.tml vfr viaframe +application/vnd.tmobile-livetv tmo +application/vnd.tri.onesource +application/vnd.trid.tpt tpt +application/vnd.triscape.mxs mxs +application/vnd.trueapp tra +application/vnd.truedoc +# cab: application/vnd.ms-cab-compressed +application/vnd.ubisoft.webplayer +application/vnd.ufdl ufdl ufd frm +application/vnd.uiq.theme utz +application/vnd.umajin umj +application/vnd.unity unityweb +application/vnd.uoml+xml uoml uo +application/vnd.uplanet.alert +application/vnd.uplanet.alert-wbxml +application/vnd.uplanet.bearer-choice +application/vnd.uplanet.bearer-choice-wbxml +application/vnd.uplanet.cacheop +application/vnd.uplanet.cacheop-wbxml +application/vnd.uplanet.channel +application/vnd.uplanet.channel-wbxml +application/vnd.uplanet.list +application/vnd.uplanet.list-wbxml +application/vnd.uplanet.listcmd +application/vnd.uplanet.listcmd-wbxml +application/vnd.uplanet.signal +application/vnd.uri-map urim urimap +application/vnd.valve.source.material vmt +application/vnd.vcx vcx +# sxi: application/vnd.sun.xml.impress +application/vnd.vd-study mxi study-inter model-inter +# mcd: application/vnd.mcd +application/vnd.vectorworks vwx +application/vnd.vel+json +application/vnd.verimatrix.vcas +application/vnd.vidsoft.vidconference vsc +application/vnd.visio vsd vst vsw vss +application/vnd.visionary vis +# vsc: application/vnd.vidsoft.vidconference +application/vnd.vividence.scriptfile +application/vnd.vsf vsf +application/vnd.wap.sic sic +application/vnd.wap.slc slc +application/vnd.wap.wbxml wbxml +application/vnd.wap.wmlc wmlc +application/vnd.wap.wmlscriptc wmlsc +application/vnd.webturbo wtb +application/vnd.wfa.p2p p2p +application/vnd.wfa.wsc wsc +application/vnd.windows.devicepairing +application/vnd.wmc wmc +application/vnd.wmf.bootstrap +# nb: application/mathematica for now +application/vnd.wolfram.mathematica +application/vnd.wolfram.mathematica.package m +application/vnd.wolfram.player nbp +application/vnd.wordperfect wpd +application/vnd.wqd wqd +application/vnd.wrq-hp3000-labelled +application/vnd.wt.stf stf +application/vnd.wv.csp+xml +application/vnd.wv.csp+wbxml wv +application/vnd.wv.ssp+xml +application/vnd.xacml+json +application/vnd.xara xar +application/vnd.xfdl xfdl xfd +application/vnd.xfdl.webform +application/vnd.xmi+xml +application/vnd.xmpie.cpkg cpkg +application/vnd.xmpie.dpkg dpkg +# dpkg: application/vnd.xmpie.dpkg +application/vnd.xmpie.plan +application/vnd.xmpie.ppkg ppkg +application/vnd.xmpie.xlim xlim +application/vnd.yamaha.hv-dic hvd +application/vnd.yamaha.hv-script hvs +application/vnd.yamaha.hv-voice hvp +application/vnd.yamaha.openscoreformat osf +application/vnd.yamaha.openscoreformat.osfpvg+xml +application/vnd.yamaha.remote-setup +application/vnd.yamaha.smaf-audio saf +application/vnd.yamaha.smaf-phrase spf +application/vnd.yamaha.through-ngn +application/vnd.yamaha.tunnel-udpencap +application/vnd.yaoweme yme +application/vnd.yellowriver-custom-menu cmp +application/vnd.zul zir zirz +application/vnd.zzazz.deck+xml zaz +application/voicexml+xml vxml +application/vq-rtcp-xr +application/watcherinfo+xml wif +application/whoispp-query +application/whoispp-response +application/widget wgt +application/wita +application/wordperfect5.1 +application/wsdl+xml wsdl +application/wspolicy+xml wspolicy +# yes, this *is* IANA registered despite of x- +application/x-www-form-urlencoded +application/x400-bp +application/xacml+xml +application/xcap-att+xml xav +application/xcap-caps+xml xca +application/xcap-diff+xml xdf +application/xcap-el+xml xel +application/xcap-error+xml xer +application/xcap-ns+xml xns +application/xcon-conference-info-diff+xml +application/xcon-conference-info+xml +application/xenc+xml +application/xhtml+xml xhtml xhtm xht +# xml, xsd, rng: text/xml +application/xml +# mod: audio/x-mod +application/xml-dtd dtd +# ent: text/xml-external-parsed-entity +application/xml-external-parsed-entity +application/xml-patch+xml +application/xmpp+xml +application/xop+xml xop +application/xslt+xml xsl xslt +application/xv+xml mxml xhvml xvml xvm +application/yang yang +application/yang-data+json +application/yang-data+xml +application/yang-patch+json +application/yang-patch+xml +application/yin+xml yin +application/zip zip +application/zlib +audio/1d-interleaved-parityfec +audio/32kadpcm 726 +# 3gp, 3gpp: video/3gpp +audio/3gpp +# 3g2, 3gpp2: video/3gpp2 +audio/3gpp2 +audio/ac3 ac3 +audio/AMR amr +audio/AMR-WB awb +audio/amr-wb+ +audio/aptx +audio/asc acn +# aa3, omg: audio/ATRAC3 +audio/ATRAC-ADVANCED-LOSSLESS aal +# aa3, omg: audio/ATRAC3 +audio/ATRAC-X atx +audio/ATRAC3 at3 aa3 omg +audio/basic au snd +audio/BV16 +audio/BV32 +audio/clearmode +audio/CN +audio/DAT12 +audio/dls dls +audio/dsr-es201108 +audio/dsr-es202050 +audio/dsr-es202211 +audio/dsr-es202212 +audio/DV +audio/DVI4 +audio/eac3 +audio/encaprtp +audio/EVRC evc +# qcp: audio/qcelp +audio/EVRC-QCP +audio/EVRC0 +audio/EVRC1 +audio/EVRCB evb +audio/EVRCB0 +audio/EVRCB1 +audio/EVRCNW enw +audio/EVRCNW0 +audio/EVRCNW1 +audio/EVRCWB evw +audio/EVRCWB0 +audio/EVRCWB1 +audio/EVS +audio/example +audio/fwdred +audio/G711-0 +audio/G719 +audio/G722 +audio/G7221 +audio/G723 +audio/G726-16 +audio/G726-24 +audio/G726-32 +audio/G726-40 +audio/G728 +audio/G729 +audio/G7291 +audio/G729D +audio/G729E +audio/GSM +audio/GSM-EFR +audio/GSM-HR-08 +audio/iLBC lbc +audio/ip-mr_v2.5 +# wav: audio/x-wav +audio/L16 l16 +audio/L20 +audio/L24 +audio/L8 +audio/LPC +audio/MELP +audio/MELP600 +audio/MELP1200 +audio/MELP2400 +audio/mobile-xmf mxmf +# mp4, mpg4: video/mp4, see RFC 4337 +audio/mp4 m4a +audio/MP4A-LATM +audio/MPA +audio/mpa-robust +audio/mpeg mp3 mpga mp1 mp2 +audio/mpeg4-generic +audio/ogg oga ogg opus spx +audio/opus +audio/parityfec +audio/PCMA +audio/PCMA-WB +audio/PCMU +audio/PCMU-WB +audio/prs.sid sid psid +audio/qcelp qcp +audio/raptorfec +audio/RED +audio/rtp-enc-aescm128 +audio/rtp-midi +audio/rtploopback +audio/rtx +audio/SMV smv +# qcp: audio/qcelp, see RFC 3625 +audio/SMV-QCP +audio/SMV0 +# mid: audio/midi +audio/sp-midi +audio/speex +audio/t140c +audio/t38 +audio/telephone-event +audio/tone +audio/UEMCLIP +audio/ulpfec +audio/VDVI +audio/VMR-WB +audio/vnd.3gpp.iufp +audio/vnd.4SB +audio/vnd.audikoz koz +audio/vnd.CELP +audio/vnd.cisco.nse +audio/vnd.cmles.radio-events +audio/vnd.cns.anp1 +audio/vnd.cns.inf1 +audio/vnd.dece.audio uva uvva +audio/vnd.digital-winds eol +audio/vnd.dlna.adts +audio/vnd.dolby.heaac.1 +audio/vnd.dolby.heaac.2 +audio/vnd.dolby.mlp mlp +audio/vnd.dolby.mps +audio/vnd.dolby.pl2 +audio/vnd.dolby.pl2x +audio/vnd.dolby.pl2z +audio/vnd.dolby.pulse.1 +audio/vnd.dra +# wav: audio/x-wav, cpt: application/mac-compactpro +audio/vnd.dts dts +audio/vnd.dts.hd dtshd +# dvb: video/vnd.dvb.file +audio/vnd.dvb.file +audio/vnd.everad.plj plj +# rm: audio/x-pn-realaudio +audio/vnd.hns.audio +audio/vnd.lucent.voice lvp +audio/vnd.ms-playready.media.pya pya +# mxmf: audio/mobile-xmf +audio/vnd.nokia.mobile-xmf +audio/vnd.nortel.vbk vbk +audio/vnd.nuera.ecelp4800 ecelp4800 +audio/vnd.nuera.ecelp7470 ecelp7470 +audio/vnd.nuera.ecelp9600 ecelp9600 +audio/vnd.octel.sbc +# audio/vnd.qcelp deprecated in favour of audio/qcelp +audio/vnd.rhetorex.32kadpcm +audio/vnd.rip rip +audio/vnd.sealedmedia.softseal.mpeg smp3 smp s1m +audio/vnd.vmx.cvsd +audio/vorbis +audio/vorbis-config +font/collection ttc +font/otf otf +font/sfnt +font/ttf ttf +font/woff woff +font/woff2 woff2 +image/bmp bmp dib +image/cgm cgm +image/dicom-rle drle +image/emf emf +image/example +image/fits fits fit fts +image/g3fax +image/gif gif +image/ief ief +image/jls jls +image/jp2 jp2 jpg2 +image/jpeg jpg jpeg jpe jfif +image/jpm jpm jpgm +image/jpx jpx jpf +image/ktx ktx +image/naplps +image/png png +image/prs.btif btif btf +image/prs.pti pti +image/pwg-raster +image/svg+xml svg svgz +image/t38 t38 +image/tiff tiff tif +image/tiff-fx tfx +image/vnd.adobe.photoshop psd +image/vnd.airzip.accelerator.azv azv +image/vnd.cns.inf2 +image/vnd.dece.graphic uvi uvvi uvg uvvg +image/vnd.djvu djvu djv +# sub: text/vnd.dvb.subtitle +image/vnd.dvb.subtitle +image/vnd.dwg dwg +image/vnd.dxf dxf +image/vnd.fastbidsheet fbs +image/vnd.fpx fpx +image/vnd.fst fst +image/vnd.fujixerox.edmics-mmr mmr +image/vnd.fujixerox.edmics-rlc rlc +image/vnd.globalgraphics.pgb pgb +image/vnd.microsoft.icon ico +image/vnd.mix +image/vnd.mozilla.apng apng +image/vnd.ms-modi mdi +image/vnd.net-fpx +image/vnd.radiance hdr rgbe xyze +image/vnd.sealed.png spng spn s1n +image/vnd.sealedmedia.softseal.gif sgif sgi s1g +image/vnd.sealedmedia.softseal.jpg sjpg sjp s1j +image/vnd.svf +image/vnd.tencent.tap tap +image/vnd.valve.source.texture vtf +image/vnd.wap.wbmp wbmp +image/vnd.xiff xif +image/vnd.zbrush.pcx pcx +image/wmf wmf +message/CPIM +message/delivery-status +message/disposition-notification +message/example +message/external-body +message/feedback-report +message/global u8msg +message/global-delivery-status u8dsn +message/global-disposition-notification u8mdn +message/global-headers u8hdr +message/http +# cl: application/simple-filter+xml +message/imdn+xml +# message/news obsoleted by message/rfc822 +message/partial +message/rfc822 eml mail art +message/s-http +message/sip +message/sipfrag +message/tracking-status +message/vnd.si.simp +# wsc: application/vnd.wfa.wsc +message/vnd.wfa.wsc +model/example +model/gltf+json gltf +model/iges igs iges +model/mesh msh mesh silo +model/vnd.collada+xml dae +model/vnd.dwf dwf +# 3dml, 3dm: text/vnd.in3d.3dml +model/vnd.flatland.3dml +model/vnd.gdl gdl gsm win dor lmp rsm msm ism +model/vnd.gs-gdl +model/vnd.gtw gtw +model/vnd.moml+xml moml +model/vnd.mts mts +model/vnd.opengex ogex +model/vnd.parasolid.transmit.binary x_b xmt_bin +model/vnd.parasolid.transmit.text x_t xmt_txt +model/vnd.rosette.annotated-data-model +model/vnd.valve.source.compiled-map bsp +model/vnd.vtu vtu +model/vrml wrl vrml +# x3db: model/x3d+xml +model/x3d+fastinfoset +# x3d: application/vnd.hzn-3d-crossword +model/x3d+xml x3db +model/x3d-vrml x3dv x3dvz +multipart/alternative +multipart/appledouble +multipart/byteranges +multipart/digest +multipart/encrypted +multipart/form-data +multipart/header-set +multipart/mixed +multipart/parallel +multipart/related +multipart/report +multipart/signed +multipart/vnd.bint.med-plus bmed +multipart/voice-message vpm +multipart/x-mixed-replace +text/1d-interleaved-parityfec +text/cache-manifest appcache manifest +text/calendar ics ifb +text/css css +text/csv csv +text/csv-schema csvs +text/directory +text/dns soa zone +text/encaprtp +# text/ecmascript obsoleted by application/ecmascript +text/enriched +text/example +text/fwdred +text/grammar-ref-list +text/html html htm +# text/javascript obsoleted by application/javascript +text/jcr-cnd cnd +text/markdown markdown md +text/mizar miz +text/n3 n3 +text/parameters +text/parityfec +text/plain txt asc text pm el c h cc hh cxx hxx f90 conf log +text/provenance-notation provn +text/prs.fallenstein.rst rst +text/prs.lines.tag tag dsc +text/prs.prop.logic +text/raptorfec +text/RED +text/rfc822-headers +text/richtext rtx +# rtf: application/rtf +text/rtf +text/rtp-enc-aescm128 +text/rtploopback +text/rtx +text/sgml sgml sgm +text/strings +text/t140 +text/tab-separated-values tsv +text/troff t tr roff +text/turtle ttl +text/ulpfec +text/uri-list uris uri +text/vcard vcf vcard +text/vnd.a a +text/vnd.abc abc +text/vnd.ascii-art ascii +# curl: application/vnd.curl +text/vnd.curl +text/vnd.debian.copyright copyright +text/vnd.DMClientScript dms +text/vnd.dvb.subtitle sub +text/vnd.esmertec.theme-descriptor jtd +text/vnd.fly fly +text/vnd.fmi.flexstor flx +text/vnd.graphviz gv dot +text/vnd.in3d.3dml 3dml 3dm +text/vnd.in3d.spot spot spo +text/vnd.IPTC.NewsML +text/vnd.IPTC.NITF +text/vnd.latex-z +text/vnd.motorola.reflex +text/vnd.ms-mediapackage mpf +text/vnd.net2phone.commcenter.command ccc +text/vnd.radisys.msml-basic-layout +text/vnd.si.uricatalogue uric +text/vnd.sun.j2me.app-descriptor jad +text/vnd.trolltech.linguist ts +text/vnd.wap.si si +text/vnd.wap.sl sl +text/vnd.wap.wml wml +text/vnd.wap.wmlscript wmls +text/xml xml xsd rng +text/xml-external-parsed-entity ent +video/1d-interleaved-parityfec +video/3gpp 3gp 3gpp +video/3gpp2 3g2 3gpp2 +video/3gpp-tt +video/BMPEG +video/BT656 +video/CelB +video/DV +video/encaprtp +video/example +video/H261 +video/H263 +video/H263-1998 +video/H263-2000 +video/H264 +video/H264-RCDO +video/H264-SVC +video/H265 +video/iso.segment m4s +video/JPEG +video/jpeg2000 +video/mj2 mj2 mjp2 +video/MP1S +video/MP2P +video/MP2T +video/mp4 mp4 mpg4 m4v +video/MP4V-ES +video/mpeg mpeg mpg mpe m1v m2v +video/mpeg4-generic +video/MPV +video/nv +video/ogg ogv +video/parityfec +video/pointer +video/quicktime mov qt +video/raptorfec +video/raw +video/rtp-enc-aescm128 +video/rtploopback +video/rtx +video/SMPTE292M +video/ulpfec +video/vc1 +video/vnd.CCTV +video/vnd.dece.hd uvh uvvh +video/vnd.dece.mobile uvm uvvm +video/vnd.dece.mp4 uvu uvvu +video/vnd.dece.pd uvp uvvp +video/vnd.dece.sd uvs uvvs +video/vnd.dece.video uvv uvvv +video/vnd.directv.mpeg +video/vnd.directv.mpeg-tts +video/vnd.dlna.mpeg-tts +video/vnd.dvb.file dvb +video/vnd.fvt fvt +# rm: audio/x-pn-realaudio +video/vnd.hns.video +video/vnd.iptvforum.1dparityfec-1010 +video/vnd.iptvforum.1dparityfec-2005 +video/vnd.iptvforum.2dparityfec-1010 +video/vnd.iptvforum.2dparityfec-2005 +video/vnd.iptvforum.ttsavc +video/vnd.iptvforum.ttsmpeg2 +video/vnd.motorola.video +video/vnd.motorola.videop +video/vnd.mpegurl mxu m4u +video/vnd.ms-playready.media.pyv pyv +video/vnd.nokia.interleaved-multimedia nim +video/vnd.nokia.videovoip +# mp4: video/mp4 +video/vnd.objectvideo +video/vnd.radgamettools.bink bik bk2 +video/vnd.radgamettools.smacker smk +video/vnd.sealed.mpeg1 smpg s11 +# smpg: video/vnd.sealed.mpeg1 +video/vnd.sealed.mpeg4 s14 +video/vnd.sealed.swf sswf ssw +video/vnd.sealedmedia.softseal.mov smov smo s1q +# uvu, uvvu: video/vnd.dece.mp4 +video/vnd.uvvu.mp4 +video/vnd.vivo viv +video/VP8 + +# Non-IANA types + +application/mac-compactpro cpt +application/metalink+xml metalink +application/owl+xml owx +application/rss+xml rss +application/vnd.android.package-archive apk +application/vnd.oma.dd+xml dd +application/vnd.oma.drm.content dcf +# odf: application/vnd.oasis.opendocument.formula +application/vnd.oma.drm.dcf o4a o4v +application/vnd.oma.drm.message dm +application/vnd.oma.drm.rights+wbxml drc +application/vnd.oma.drm.rights+xml dr +application/vnd.sun.xml.calc sxc +application/vnd.sun.xml.calc.template stc +application/vnd.sun.xml.draw sxd +application/vnd.sun.xml.draw.template std +application/vnd.sun.xml.impress sxi +application/vnd.sun.xml.impress.template sti +application/vnd.sun.xml.math sxm +application/vnd.sun.xml.writer sxw +application/vnd.sun.xml.writer.global sxg +application/vnd.sun.xml.writer.template stw +application/vnd.symbian.install sis +application/vnd.wap.mms-message mms +application/x-annodex anx +application/x-bcpio bcpio +application/x-bittorrent torrent +application/x-bzip2 bz2 +application/x-cdlink vcd +application/x-chrome-extension crx +application/x-cpio cpio +application/x-csh csh +application/x-director dcr dir dxr +application/x-dvi dvi +application/x-futuresplash spl +application/x-gtar gtar +application/x-hdf hdf +application/x-java-archive jar +application/x-java-jnlp-file jnlp +application/x-java-pack200 pack +application/x-killustrator kil +application/x-latex latex +application/x-netcdf nc cdf +application/x-perl pl +application/x-rpm rpm +application/x-sh sh +application/x-shar shar +application/x-stuffit sit +application/x-sv4cpio sv4cpio +application/x-sv4crc sv4crc +application/x-tar tar +application/x-tcl tcl +application/x-tex tex +application/x-texinfo texinfo texi +application/x-troff-man man 1 2 3 4 5 6 7 8 +application/x-troff-me me +application/x-troff-ms ms +application/x-ustar ustar +application/x-wais-source src +application/x-xpinstall xpi +application/x-xspf+xml xspf +application/x-xz xz +audio/midi mid midi kar +audio/x-aiff aif aiff aifc +audio/x-annodex axa +audio/x-flac flac +audio/x-matroska mka +audio/x-mod mod ult uni m15 mtm 669 med +audio/x-mpegurl m3u +audio/x-ms-wax wax +audio/x-ms-wma wma +audio/x-pn-realaudio ram rm +audio/x-realaudio ra +audio/x-s3m s3m +audio/x-stm stm +audio/x-wav wav +chemical/x-xyz xyz +image/webp webp +image/x-cmu-raster ras +image/x-portable-anymap pnm +image/x-portable-bitmap pbm +image/x-portable-graymap pgm +image/x-portable-pixmap ppm +image/x-rgb rgb +image/x-targa tga +image/x-xbitmap xbm +image/x-xpixmap xpm +image/x-xwindowdump xwd +text/html-sandboxed sandboxed +text/x-pod pod +text/x-setext etx +video/webm webm +video/x-annodex axv +video/x-flv flv +video/x-javafx fxm +video/x-matroska mkv +video/x-matroska-3d mk3d +video/x-ms-asf asx +video/x-ms-wm wm +video/x-ms-wmv wmv +video/x-ms-wmx wmx +video/x-ms-wvx wvx +video/x-msvideo avi +video/x-sgi-movie movie +x-conference/x-cooltalk ice +x-epoc/x-sisx-app sisx From fc9a23b7574da9609e813330244fb45f592f919a Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Mon, 4 Mar 2024 05:36:33 -0500 Subject: [PATCH 2/5] Expose Apache server --- controllers/octavia_controller.go | 85 +++++++++++++++++++++----- pkg/octavia/const.go | 3 + pkg/octavia/image_upload_deployment.go | 16 +---- 3 files changed, 74 insertions(+), 30 deletions(-) diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index a9f3f5cd..4f576df7 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -38,6 +38,7 @@ import ( common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" @@ -50,7 +51,6 @@ import ( rbacv1 "k8s.io/api/rbac/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" - k8s_labels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" @@ -681,7 +681,71 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav } readyCount := depl.GetDeployment().Status.ReadyReplicas if readyCount > 0 { - urlMap, err := r.getLocalImageURLs(ctx, helper, instance) + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + service.AnnotationEndpointKey: "internal", + }, + ) + + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: "octavia-image-upload-internal", + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Ports: []corev1.ServicePort{ + { + Name: "octavia-image-upload-internal", + Port: octavia.ApacheInternalPort, + Protocol: corev1.ProtocolTCP, + }, + }, + }), + 5, + nil, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: "internal", + }) + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + + ctrlResult, err := svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.ExposeServiceReadyRunningMessage)) + return ctrlResult, nil + } + endpoint, err := svc.GetAPIEndpoint(nil, nil, "") + if err != nil { + return ctrl.Result{}, err + } + + urlMap, err := r.getLocalImageURLs(ctx, helper, endpoint) if err != nil { return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, err } @@ -840,19 +904,10 @@ func (r *OctaviaReconciler) ensureDB( func (r *OctaviaReconciler) getLocalImageURLs( ctx context.Context, helper *helper.Helper, - instance *octaviav1.Octavia, + endpoint string, ) ([]octavia.OctaviaAmphoraImage, error) { - serviceLabels := map[string]string{common.AppSelector: instance.Name + "-deployment"} - podSelectorString := k8s_labels.Set(serviceLabels).String() - pods, err := helper.GetKClient().CoreV1().Pods( - instance.Namespace).List(ctx, metav1.ListOptions{LabelSelector: podSelectorString}) - if err != nil { - return nil, err - } - host := pods.Items[0].Status.HostIP - // Get the list of images and their hashes - listUrl := fmt.Sprintf("http://%s:%d/octavia-amphora-images.sha256sum", host, 8080) + listUrl := fmt.Sprintf("%s/octavia-amphora-images.sha256sum", endpoint) resp, err := http.Get(listUrl) if err != nil { @@ -867,7 +922,7 @@ func (r *OctaviaReconciler) getLocalImageURLs( name, _ := strings.CutSuffix(fields[1], ".qcow2") ret = append(ret, octavia.OctaviaAmphoraImage{ Name: name, - URL: fmt.Sprintf("http://%s:%d/%s", host, 8080, fields[1]), + URL: fmt.Sprintf("%s/%s", endpoint, fields[1]), Checksum: fields[0], }) } @@ -927,7 +982,7 @@ func (r *OctaviaReconciler) generateServiceConfigMaps( ), } templateParameters["ServiceUser"] = instance.Spec.ServiceUser - templateParameters["ApachePort"] = 8080 + templateParameters["ApachePort"] = octavia.ApacheInternalPort cms := []util.Template{ // ScriptsConfigMap diff --git a/pkg/octavia/const.go b/pkg/octavia/const.go index e7f4f78b..056ef327 100644 --- a/pkg/octavia/const.go +++ b/pkg/octavia/const.go @@ -43,4 +43,7 @@ const ( OctaviaPublicPort int32 = 9876 // OctaviaInternalPort - OctaviaInternalPort int32 = 9876 + + // ApacheInternalPort - + ApacheInternalPort int32 = 8080 ) diff --git a/pkg/octavia/image_upload_deployment.go b/pkg/octavia/image_upload_deployment.go index b0886ee3..7bfc7bc0 100644 --- a/pkg/octavia/image_upload_deployment.go +++ b/pkg/octavia/image_upload_deployment.go @@ -89,7 +89,6 @@ func ImageUploadDeployment( ) *appsv1.Deployment { initVolumeMounts := getInitVolumeMounts() - // TODO(gthiemonge) healthchecks? args := []string{"-c", ServiceCommand} serviceName := fmt.Sprintf("%s-image-upload", ServiceName) @@ -110,8 +109,6 @@ func ImageUploadDeployment( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.RbacResourceName(), - // TODO(gthiemonge) Expose service on Pod - HostNetwork: true, Containers: []corev1.Container{ { Name: "octavia-amphora-httpd", @@ -122,19 +119,8 @@ func ImageUploadDeployment( Image: instance.Spec.ApacheContainerImage, VolumeMounts: getVolumeMounts("octavia-image-upload"), Resources: instance.Spec.Resources, - // TODO(gthiemonge) Probes? + // TODO(gthiemonge) do we need probes? }, - //{ - // Name: serviceName, - // Command: []string{ - // "/bin/bash", - // }, - // Image: instance.Spec.AmphoraImageContainerImage, - // SecurityContext: &corev1.SecurityContext{ - // RunAsUser: &runAsUser, - // }, - // VolumeMounts: getVolumeMounts("octavia-image-upload"), - //}, }, Volumes: getVolumes(instance.Name), }, From b624b7b94baead3033cd1788474e79ff1c21a39f Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Mon, 4 Mar 2024 06:02:18 -0500 Subject: [PATCH 3/5] Move image related stuff to a new reconcileAmphoraImages function --- controllers/octavia_controller.go | 260 +++++++++++++++++------------- 1 file changed, 151 insertions(+), 109 deletions(-) diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index 4f576df7..99909ff8 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -177,6 +177,8 @@ func (r *OctaviaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re condition.UnknownCondition(condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), condition.UnknownCondition(octaviav1.OctaviaAPIReadyCondition, condition.InitReason, octaviav1.OctaviaAPIReadyInitMessage), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), amphoraControllerInitCondition(octaviav1.HealthManager), amphoraControllerInitCondition(octaviav1.Housekeeping), amphoraControllerInitCondition(octaviav1.Worker), @@ -214,6 +216,7 @@ func (r *OctaviaReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Owns(&corev1.Service{}). Owns(&rabbitmqv1.TransportURL{}). Owns(&redisv1.Redis{}). Complete(r) @@ -661,115 +664,12 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav return ctrl.Result{}, err } - // Amphora Image deployment - if instance.Spec.AmphoraImageContainerImage != "" { - hash, err := util.ObjectHash(instance.Spec.AmphoraImageContainerImage) - if err != nil { - return ctrl.Result{}, err - } - if hash != instance.Status.Hash[octaviav1.ImageUploadHash] { - Log.Info("Initializing amphora image upload deployment") - depl := deployment.NewDeployment( - octavia.ImageUploadDeployment(instance, serviceLabels, serviceAnnotations), - time.Duration(5)*time.Second, - ) - ctrlResult, err = depl.CreateOrPatch(ctx, helper) - if err != nil { - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil - } - readyCount := depl.GetDeployment().Status.ReadyReplicas - if readyCount > 0 { - exportLabels := util.MergeStringMaps( - serviceLabels, - map[string]string{ - service.AnnotationEndpointKey: "internal", - }, - ) - - svc, err := service.NewService( - service.GenericService(&service.GenericServiceDetails{ - Name: "octavia-image-upload-internal", - Namespace: instance.Namespace, - Labels: exportLabels, - Selector: serviceLabels, - Ports: []corev1.ServicePort{ - { - Name: "octavia-image-upload-internal", - Port: octavia.ApacheInternalPort, - Protocol: corev1.ProtocolTCP, - }, - }, - }), - 5, - nil, - ) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.ExposeServiceReadyErrorMessage, - err.Error())) - - return ctrl.Result{}, err - } - svc.AddAnnotation(map[string]string{ - service.AnnotationEndpointKey: "internal", - }) - svc.AddAnnotation(map[string]string{ - service.AnnotationIngressCreateKey: "false", - }) - - ctrlResult, err := svc.CreateOrPatch(ctx, helper) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.ExposeServiceReadyErrorMessage, - err.Error())) - - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.ExposeServiceReadyRunningMessage)) - return ctrlResult, nil - } - endpoint, err := svc.GetAPIEndpoint(nil, nil, "") - if err != nil { - return ctrl.Result{}, err - } - - urlMap, err := r.getLocalImageURLs(ctx, helper, endpoint) - if err != nil { - return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, err - } - - ok, err := octavia.EnsureAmphoraImages(ctx, instance, &r.Log, helper, urlMap) - if err != nil { - return ctrl.Result{}, err - } - if !ok { - // Images are not ready - return ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil - } - Log.Info(fmt.Sprintf("Setting image upload hash - %s", hash)) - instance.Status.Hash[octaviav1.ImageUploadHash] = hash - - Log.Info("Deleting amphora image upload deployment") - depl.Delete(ctx, helper) - } - } - } else { - if instance.Status.Hash[octaviav1.ImageUploadHash] != "" { - Log.Info("Reseting image upload hash") - instance.Status.Hash[octaviav1.ImageUploadHash] = "" - } + ctrlResult, err = r.reconcileAmphoraImages(ctx, instance, helper, serviceLabels, serviceAnnotations) + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + if err != nil { + return ctrl.Result{}, err } // create Deployment - end @@ -901,6 +801,148 @@ func (r *OctaviaReconciler) ensureDB( // create service DB - end } +func (r *OctaviaReconciler) reconcileAmphoraImages( + ctx context.Context, + instance *octaviav1.Octavia, + helper *helper.Helper, + serviceLabels map[string]string, + serviceAnnotations map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + if instance.Spec.AmphoraImageContainerImage == "" { + if instance.Status.Hash[octaviav1.ImageUploadHash] != "" { + Log.Info("Reseting image upload hash") + instance.Status.Hash[octaviav1.ImageUploadHash] = "" + } + return ctrl.Result{}, nil + } + + hash, err := util.ObjectHash(instance.Spec.AmphoraImageContainerImage) + if err != nil { + return ctrl.Result{}, err + } + if hash == instance.Status.Hash[octaviav1.ImageUploadHash] { + // No change + return ctrl.Result{}, nil + } + + Log.Info("Initializing amphora image upload deployment") + depl := deployment.NewDeployment( + octavia.ImageUploadDeployment(instance, serviceLabels, serviceAnnotations), + time.Duration(5)*time.Second, + ) + ctrlResult, err := depl.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrlResult, nil + } + + readyCount := depl.GetDeployment().Status.ReadyReplicas + if readyCount == 0 { + // Not ready, wait for the next loop + return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, nil + } + + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + service.AnnotationEndpointKey: "internal", + }, + ) + + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: "octavia-image-upload-internal", + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Ports: []corev1.ServicePort{ + { + Name: "octavia-image-upload-internal", + Port: octavia.ApacheInternalPort, + Protocol: corev1.ProtocolTCP, + }, + }, + }), + 5, + nil, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: "internal", + }) + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + + ctrlResult, err = svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.ExposeServiceReadyRunningMessage)) + return ctrlResult, nil + } + endpoint, err := svc.GetAPIEndpoint(nil, nil, "") + if err != nil { + return ctrl.Result{}, err + } + + urlMap, err := r.getLocalImageURLs(ctx, helper, endpoint) + if err != nil { + return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, err + } + + ok, err := octavia.EnsureAmphoraImages(ctx, instance, &r.Log, helper, urlMap) + if err != nil { + return ctrl.Result{}, err + } + if !ok { + // Images are not ready + return ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil + } + Log.Info(fmt.Sprintf("Setting image upload hash - %s", hash)) + instance.Status.Hash[octaviav1.ImageUploadHash] = hash + + // Tasks are successfull, the deployment can be deleted + Log.Info("Deleting amphora image upload deployment") + depl.Delete(ctx, helper) + + return ctrl.Result{}, nil +} + func (r *OctaviaReconciler) getLocalImageURLs( ctx context.Context, helper *helper.Helper, From 8b413d909873a460edac00b546049cb0c2aba616 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Mon, 4 Mar 2024 09:25:14 -0500 Subject: [PATCH 4/5] Using unique service label for image upload --- controllers/octavia_controller.go | 13 +++++++++---- pkg/octavia/image_upload_deployment.go | 4 +--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index 99909ff8..f545643a 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -664,7 +664,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav return ctrl.Result{}, err } - ctrlResult, err = r.reconcileAmphoraImages(ctx, instance, helper, serviceLabels, serviceAnnotations) + ctrlResult, err = r.reconcileAmphoraImages(ctx, instance, helper) if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil } @@ -805,8 +805,6 @@ func (r *OctaviaReconciler) reconcileAmphoraImages( ctx context.Context, instance *octaviav1.Octavia, helper *helper.Helper, - serviceLabels map[string]string, - serviceAnnotations map[string]string, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) @@ -827,9 +825,13 @@ func (r *OctaviaReconciler) reconcileAmphoraImages( return ctrl.Result{}, nil } + serviceLabels := map[string]string{ + common.AppSelector: octavia.ServiceName + "-image", + } + Log.Info("Initializing amphora image upload deployment") depl := deployment.NewDeployment( - octavia.ImageUploadDeployment(instance, serviceLabels, serviceAnnotations), + octavia.ImageUploadDeployment(instance, serviceLabels), time.Duration(5)*time.Second, ) ctrlResult, err := depl.CreateOrPatch(ctx, helper) @@ -853,6 +855,7 @@ func (r *OctaviaReconciler) reconcileAmphoraImages( readyCount := depl.GetDeployment().Status.ReadyReplicas if readyCount == 0 { // Not ready, wait for the next loop + Log.Info("Image Upload Pod not ready") return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, nil } @@ -922,6 +925,7 @@ func (r *OctaviaReconciler) reconcileAmphoraImages( urlMap, err := r.getLocalImageURLs(ctx, helper, endpoint) if err != nil { + Log.Info(fmt.Sprintf("Cannot get amphora image list: %s", err)) return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, err } @@ -931,6 +935,7 @@ func (r *OctaviaReconciler) reconcileAmphoraImages( } if !ok { // Images are not ready + Log.Info("Waiting for amphora images to be ready") return ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil } Log.Info(fmt.Sprintf("Setting image upload hash - %s", hash)) diff --git a/pkg/octavia/image_upload_deployment.go b/pkg/octavia/image_upload_deployment.go index 7bfc7bc0..080be7b7 100644 --- a/pkg/octavia/image_upload_deployment.go +++ b/pkg/octavia/image_upload_deployment.go @@ -85,7 +85,6 @@ func getVolumeMounts(serviceName string) []corev1.VolumeMount { func ImageUploadDeployment( instance *octaviav1.Octavia, labels map[string]string, - annotations map[string]string, ) *appsv1.Deployment { initVolumeMounts := getInitVolumeMounts() @@ -104,8 +103,7 @@ func ImageUploadDeployment( }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Annotations: annotations, - Labels: labels, + Labels: labels, }, Spec: corev1.PodSpec{ ServiceAccountName: instance.RbacResourceName(), From fcbc6187551e550019cfcd37b0a52875df7ef345 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Mon, 4 Mar 2024 11:30:16 -0500 Subject: [PATCH 5/5] Fix env var for Apache container --- api/v1beta1/octavia_types.go | 2 +- config/default/manager_default_images.yaml | 2 +- tests/kuttl/common/assert_sample_deployment.yaml | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index 1afa5870..4c6cdb12 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -319,7 +319,7 @@ func SetupDefaults() { HousekeepingContainerImageURL: util.GetEnvVar("RELATED_IMAGE_OCTAVIA_HOUSEKEEPING_IMAGE_URL_DEFAULT", OctaviaHousekeepingContainerImage), HealthManagerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_OCTAVIA_HEALTHMANAGER_IMAGE_URL_DEFAULT", OctaviaHealthManagerContainerImage), WorkerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_OCTAVIA_WORKER_IMAGE_URL_DEFAULT", OctaviaWorkerContainerImage), - ApacheContainerImageURL: util.GetEnvVar("RELATED_IMAGE_APACHE_IMAGE_URL_DEFAULT", ApacheContainerImage), + ApacheContainerImageURL: util.GetEnvVar("RELATED_IMAGE_OCTAVIA_APACHE_IMAGE_URL_DEFAULT", ApacheContainerImage), // No default for AmphoraImageContainerImageURL } diff --git a/config/default/manager_default_images.yaml b/config/default/manager_default_images.yaml index 6836c990..f6e233d5 100644 --- a/config/default/manager_default_images.yaml +++ b/config/default/manager_default_images.yaml @@ -19,5 +19,5 @@ spec: value: quay.io/podified-antelope-centos9/openstack-octavia-health-manager:current-podified - name: RELATED_IMAGE_OCTAVIA_WORKER_IMAGE_URL_DEFAULT value: quay.io/podified-antelope-centos9/openstack-octavia-worker:current-podified - - name: RELATED_IMAGE_APACHE_IMAGE_URL_DEFAULT + - name: RELATED_IMAGE_OCTAVIA_APACHE_IMAGE_URL_DEFAULT value: registry.redhat.io/rhel8/httpd-24:latest diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index d09ed5dd..5fc0a6fd 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -239,10 +239,13 @@ commands: WORKER) template='{{.spec.octaviaWorker.containerImage}}' ;; + APACHE) + template='{{.spec.apacheContainerImage}}' + ;; esac SERVICE_IMAGE=$(oc get -n $NAMESPACE octavia octavia -o go-template="$template") if [ "$SERVICE_IMAGE" != "$IMG_FROM_ENV" ]; then - echo "$NAME image does not equal $IMG_FROM_ENV" + echo "$NAME image does not equal $IMG_FROM_ENV (its current value is $SERVICE_IMAGE)" exit 1 fi fi