From d82301ec75c5edd6edf0ae1ae74fa980090b279b Mon Sep 17 00:00:00 2001 From: Enxebre Date: Thu, 26 Sep 2019 13:19:18 +0200 Subject: [PATCH 01/14] Report machineapierros.InvalidMachineConfiguration when appropriate This is to satisfy contract with the machine controller so it can react appropriately and signal a machine as failed with the given error message when an InvalidMachineConfiguration error is reported, main cases are: - The spec is invalid: Unable to be unmarshalled Missing expected labels by design Referencing non existing kubernetes resources - The input values provoke a client error 4xx response from the cloud call to create a machine. --- pkg/actuators/machine/actuator.go | 93 ++++++++++---------- pkg/actuators/machine/actuator_test.go | 112 +++++++++++++++++++++---- pkg/actuators/machine/instances.go | 31 +++++-- pkg/client/client.go | 10 ++- 4 files changed, 175 insertions(+), 71 deletions(-) diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index e68bb341ff..e097f5d6e1 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -21,29 +21,25 @@ import ( "fmt" "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang/glog" - + clusterv1 "github.com/openshift/cluster-api/pkg/apis/cluster/v1alpha1" + machinev1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" + clustererror "github.com/openshift/cluster-api/pkg/controller/error" + machinecontroller "github.com/openshift/cluster-api/pkg/controller/machine" + mapierrors "github.com/openshift/cluster-api/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" + apimachineryerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" errorutil "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/klog" - - clusterv1 "github.com/openshift/cluster-api/pkg/apis/cluster/v1alpha1" - machinev1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" - clustererror "github.com/openshift/cluster-api/pkg/controller/error" - apierrors "github.com/openshift/cluster-api/pkg/errors" providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - awsclient "sigs.k8s.io/cluster-api-provider-aws/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client" - - machinecontroller "github.com/openshift/cluster-api/pkg/controller/machine" ) const ( @@ -99,12 +95,12 @@ const ( // Set corresponding event based on error. It also returns the original error // for convenience, so callers can do "return handleMachineError(...)". -func (a *Actuator) handleMachineError(machine *machinev1.Machine, err *apierrors.MachineError, eventAction string) error { +func (a *Actuator) handleMachineError(machine *machinev1.Machine, err error, eventAction string) error { if eventAction != noEventAction { - a.eventRecorder.Eventf(machine, corev1.EventTypeWarning, "Failed"+eventAction, "%v", err.Reason) + a.eventRecorder.Eventf(machine, corev1.EventTypeWarning, "Failed"+eventAction, "%v", err) } - glog.Errorf("%s: Machine error: %v", machine.Name, err.Message) + glog.Errorf("%s: Machine error: %v", machine.Name, err) return err } @@ -267,7 +263,7 @@ func (a *Actuator) updateMachineProviderConditions(machine *machinev1.Machine, c func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *machinev1.Machine) (*ec2.Instance, error) { machineProviderConfig, err := providerConfigFromMachine(machine, a.codec) if err != nil { - return nil, a.handleMachineError(machine, apierrors.InvalidMachineConfiguration("error decoding MachineProviderConfig: %v", err), createEventAction) + return nil, a.handleMachineError(machine, mapierrors.InvalidMachineConfiguration("error decoding MachineProviderConfig: %v", err), createEventAction) } credentialsSecretName := "" @@ -276,8 +272,7 @@ func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *machinev1. } awsClient, err := a.awsClientBuilder(a.client, credentialsSecretName, machine.Namespace, machineProviderConfig.Placement.Region) if err != nil { - glog.Errorf("%s: unable to obtain AWS client: %v", machine.Name, err) - return nil, a.handleMachineError(machine, apierrors.CreateMachine("error creating aws services: %v", err), createEventAction) + return nil, a.handleMachineError(machine, err, createEventAction) } // We explicitly do NOT want to remove stopped masters. @@ -300,34 +295,45 @@ func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *machinev1. } } - userData := []byte{} - if machineProviderConfig.UserDataSecret != nil { - var userDataSecret corev1.Secret - err := a.client.Get(context.Background(), client.ObjectKey{Namespace: machine.Namespace, Name: machineProviderConfig.UserDataSecret.Name}, &userDataSecret) - if err != nil { - return nil, a.handleMachineError(machine, apierrors.CreateMachine("error getting user data secret %s: %v", machineProviderConfig.UserDataSecret.Name, err), createEventAction) - } - if data, exists := userDataSecret.Data[userDataSecretKey]; exists { - userData = data - } else { - glog.Warningf("%s: Secret %v/%v does not have %q field set. Thus, no user data applied when creating an instance.", machine.Name, machine.Namespace, machineProviderConfig.UserDataSecret.Name, userDataSecretKey) - } + userData, err := a.getUserData(machine, machineProviderConfig) + if err != nil { + return nil, err } instance, err := launchInstance(machine, machineProviderConfig, userData, awsClient) if err != nil { - return nil, a.handleMachineError(machine, apierrors.CreateMachine("error launching instance: %v", err), createEventAction) + return nil, a.handleMachineError(machine, err, createEventAction) } err = a.updateLoadBalancers(awsClient, machineProviderConfig, instance, machine.Name) if err != nil { - return nil, a.handleMachineError(machine, apierrors.CreateMachine("error updating load balancers: %v", err), createEventAction) + return nil, a.handleMachineError(machine, err, createEventAction) } a.eventRecorder.Eventf(machine, corev1.EventTypeNormal, "Created", "Created Machine %v", machine.Name) return instance, nil } +func (a *Actuator) getUserData(machine *machinev1.Machine, machineProviderConfig *providerconfigv1.AWSMachineProviderConfig) ([]byte, error) { + if machineProviderConfig.UserDataSecret == nil { + return []byte{}, nil + } + + var userDataSecret corev1.Secret + err := a.client.Get(context.Background(), client.ObjectKey{Namespace: machine.Namespace, Name: machineProviderConfig.UserDataSecret.Name}, &userDataSecret) + if err != nil { + if apimachineryerrors.IsNotFound(err) { + return nil, a.handleMachineError(machine, mapierrors.InvalidMachineConfiguration("user data secret %s: %v not found", machineProviderConfig.UserDataSecret.Name, err), createEventAction) + } + return nil, a.handleMachineError(machine, mapierrors.CreateMachine("error getting user data secret %s: %v", machineProviderConfig.UserDataSecret.Name, err), createEventAction) + } + userData, exists := userDataSecret.Data[userDataSecretKey] + if !exists { + return nil, a.handleMachineError(machine, mapierrors.InvalidMachineConfiguration("%s: Secret %v/%v does not have %q field set. Thus, no user data applied when creating an instance.", machine.Name, machine.Namespace, machineProviderConfig.UserDataSecret.Name, userDataSecretKey), createEventAction) + } + return userData, nil +} + // Delete deletes a machine and updates its finalizer func (a *Actuator) Delete(context context.Context, cluster *clusterv1.Cluster, machine *machinev1.Machine) error { glog.Infof("%s: deleting machine", machine.Name) @@ -352,7 +358,7 @@ func (gl *glogLogger) Logf(format string, v ...interface{}) { func (a *Actuator) DeleteMachine(cluster *clusterv1.Cluster, machine *machinev1.Machine) error { machineProviderConfig, err := providerConfigFromMachine(machine, a.codec) if err != nil { - return a.handleMachineError(machine, apierrors.InvalidMachineConfiguration("error decoding MachineProviderConfig: %v", err), deleteEventAction) + return a.handleMachineError(machine, mapierrors.InvalidMachineConfiguration("error decoding MachineProviderConfig: %v", err), deleteEventAction) } region := machineProviderConfig.Placement.Region @@ -362,9 +368,7 @@ func (a *Actuator) DeleteMachine(cluster *clusterv1.Cluster, machine *machinev1. } client, err := a.awsClientBuilder(a.client, credentialsSecretName, machine.Namespace, region) if err != nil { - errMsg := fmt.Errorf("%s: error getting EC2 client: %v", machine.Name, err) - glog.Error(errMsg) - return errMsg + return a.handleMachineError(machine, err, deleteEventAction) } // Get all instances not terminated. @@ -382,7 +386,7 @@ func (a *Actuator) DeleteMachine(cluster *clusterv1.Cluster, machine *machinev1. terminatingInstances, err := terminateInstances(client, existingInstances) if err != nil { - return a.handleMachineError(machine, apierrors.DeleteMachine(err.Error()), noEventAction) + return a.handleMachineError(machine, mapierrors.DeleteMachine(err.Error()), noEventAction) } if len(terminatingInstances) == 1 { @@ -406,7 +410,7 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m machineProviderConfig, err := providerConfigFromMachine(machine, a.codec) if err != nil { - return a.handleMachineError(machine, apierrors.InvalidMachineConfiguration("error decoding MachineProviderConfig: %v", err), updateEventAction) + return a.handleMachineError(machine, mapierrors.InvalidMachineConfiguration("error decoding MachineProviderConfig: %v", err), updateEventAction) } region := machineProviderConfig.Placement.Region @@ -417,9 +421,7 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m } client, err := a.awsClientBuilder(a.client, credentialsSecretName, machine.Namespace, region) if err != nil { - errMsg := fmt.Errorf("%s: error getting EC2 client: %v", machine.Name, err) - glog.Error(errMsg) - return errMsg + return a.handleMachineError(machine, err, updateEventAction) } // Get all instances not terminated. existingInstances, err := getExistingInstances(machine, client) @@ -435,7 +437,7 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m if existingLen == 0 { glog.Warningf("%s: attempted to update machine but no instances found", machine.Name) - a.handleMachineError(machine, apierrors.UpdateMachine("no instance found, reason unknown"), updateEventAction) + a.handleMachineError(machine, mapierrors.UpdateMachine("no instance found, reason unknown"), updateEventAction) // Update status to clear out machine details. if err := a.updateStatus(machine, nil); err != nil { @@ -457,7 +459,7 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m err = a.updateLoadBalancers(client, machineProviderConfig, newestInstance, machine.Name) if err != nil { - a.handleMachineError(machine, apierrors.CreateMachine("Error updating load balancers: %v", err), updateEventAction) + a.handleMachineError(machine, mapierrors.CreateMachine("Error updating load balancers: %v", err), updateEventAction) return err } } else { @@ -535,9 +537,8 @@ func (a *Actuator) getMachineInstances(cluster *clusterv1.Cluster, machine *mach } client, err := a.awsClientBuilder(a.client, credentialsSecretName, machine.Namespace, region) if err != nil { - errMsg := fmt.Sprintf("%s: Error getting EC2 client: %v", machine.Name, err) - glog.Errorf(errMsg) - return nil, fmt.Errorf(errMsg) + glog.Errorf("%s: Error getting EC2 client: %v", machine.Name, err) + return nil, err } return getExistingInstances(machine, client) diff --git a/pkg/actuators/machine/actuator_test.go b/pkg/actuators/machine/actuator_test.go index a03bf4e8a9..4d7244a76b 100644 --- a/pkg/actuators/machine/actuator_test.go +++ b/pkg/actuators/machine/actuator_test.go @@ -1,27 +1,28 @@ package machine import ( + "bytes" "context" "fmt" "strings" "testing" "time" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "k8s.io/client-go/tools/record" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/golang/mock/gomock" clusterv1 "github.com/openshift/cluster-api/pkg/apis/cluster/v1alpha1" machinev1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" + machineapierrors "github.com/openshift/cluster-api/pkg/errors" + "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1" awsclient "sigs.k8s.io/cluster-api-provider-aws/pkg/client" mockaws "sigs.k8s.io/cluster-api-provider-aws/pkg/client/mock" - - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -77,7 +78,7 @@ func TestMachineEvents(t *testing.T) { operation: func(actuator *Actuator, cluster *clusterv1.Cluster, machine *machinev1.Machine) { actuator.CreateMachine(cluster, machine) }, - event: "Warning FailedCreate InvalidConfiguration", + event: "Warning FailedCreate error decoding MachineProviderConfig: unable to find machine provider config: Spec.ProviderSpec.Value is not set", }, { name: "Create machine event failed (error creating aws service)", @@ -86,7 +87,7 @@ func TestMachineEvents(t *testing.T) { operation: func(actuator *Actuator, cluster *clusterv1.Cluster, machine *machinev1.Machine) { actuator.CreateMachine(cluster, machine) }, - event: "Warning FailedCreate CreateError", + event: "Warning FailedCreate error creating aws service", }, { name: "Create machine event failed (error launching instance)", @@ -95,16 +96,16 @@ func TestMachineEvents(t *testing.T) { operation: func(actuator *Actuator, cluster *clusterv1.Cluster, machine *machinev1.Machine) { actuator.CreateMachine(cluster, machine) }, - event: "Warning FailedCreate CreateError", + event: "Warning FailedCreate error creating EC2 instance: error", }, { name: "Create machine event failed (error updating load balancers)", machine: machine, - lbErr: fmt.Errorf("error"), + lbErr: fmt.Errorf("lb error"), operation: func(actuator *Actuator, cluster *clusterv1.Cluster, machine *machinev1.Machine) { actuator.CreateMachine(cluster, machine) }, - event: "Warning FailedCreate CreateError", + event: "Warning FailedCreate lb error", }, { name: "Create machine event succeed", @@ -128,7 +129,7 @@ func TestMachineEvents(t *testing.T) { operation: func(actuator *Actuator, cluster *clusterv1.Cluster, machine *machinev1.Machine) { actuator.DeleteMachine(cluster, machine) }, - event: "Warning FailedDelete InvalidConfiguration", + event: "Warning FailedDelete error decoding MachineProviderConfig: unable to find machine provider config: Spec.ProviderSpec.Value is not set", }, { name: "Delete machine event succeed", @@ -727,3 +728,86 @@ func (m placementMatcher) Matches(input interface{}) bool { func (m placementMatcher) String() string { return fmt.Sprintf("is placement: %#v", m.placement) } + +func TestGetUserData(t *testing.T) { + machine, err := stubMachine() + if err != nil { + t.Fatal(err) + } + providerConfig := stubProviderConfig() + codec, err := providerconfigv1.NewCodec() + if err != nil { + t.Fatalf("unable to build codec: %v", err) + } + + testCases := []struct { + secret *apiv1.Secret + error error + }{ + { + secret: &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: userDataSecretName, + Namespace: defaultNamespace, + }, + Data: map[string][]byte{ + userDataSecretKey: []byte(userDataBlob), + }, + }, + error: nil, + }, + { + secret: &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "notFound", + Namespace: defaultNamespace, + }, + Data: map[string][]byte{ + userDataSecretKey: []byte(userDataBlob), + }, + }, + error: &machineapierrors.MachineError{}, + }, + { + secret: &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: userDataSecretName, + Namespace: defaultNamespace, + }, + Data: map[string][]byte{ + "badKey": []byte(userDataBlob), + }, + }, + error: &machineapierrors.MachineError{}, + }, + } + + for _, tc := range testCases { + params := ActuatorParams{ + Client: fake.NewFakeClient(tc.secret), + Codec: codec, + EventRecorder: &record.FakeRecorder{ + Events: make(chan string, 1), + }, + } + actuator, err := NewActuator(params) + if err != nil { + t.Fatalf("Could not create AWS machine actuator: %v", err) + } + userData, err := actuator.getUserData(machine, providerConfig) + if tc.error != nil { + if err == nil { + t.Fatal("Expected error") + } + _, expectMachineError := tc.error.(*machineapierrors.MachineError) + _, gotMachineError := err.(*machineapierrors.MachineError) + if expectMachineError && !gotMachineError || !expectMachineError && gotMachineError { + t.Errorf("Expected %T, got: %T", tc.error, err) + } + } else { + if compare := bytes.Compare(userData, []byte(userDataBlob)); compare != 0 { + t.Errorf("Expected: %v, got: %v", []byte(userDataBlob), userData) + } + } + } +} diff --git a/pkg/actuators/machine/instances.go b/pkg/actuators/machine/instances.go index 33295921d8..4309cf4831 100644 --- a/pkg/actuators/machine/instances.go +++ b/pkg/actuators/machine/instances.go @@ -4,11 +4,14 @@ import ( "encoding/base64" "fmt" "sort" + "strconv" + "strings" "time" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/golang/glog" - machinev1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" + mapierrors "github.com/openshift/cluster-api/pkg/errors" providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1" "github.com/aws/aws-sdk-go/aws" @@ -219,16 +222,16 @@ func getBlockDeviceMappings(blockDeviceMappings []providerconfigv1.BlockDeviceMa func launchInstance(machine *machinev1.Machine, machineProviderConfig *providerconfigv1.AWSMachineProviderConfig, userData []byte, client awsclient.Client) (*ec2.Instance, error) { amiID, err := getAMI(machineProviderConfig.AMI, client) if err != nil { - return nil, fmt.Errorf("error getting AMI: %v,", err) + return nil, mapierrors.InvalidMachineConfiguration("error getting AMI: %v", err) } securityGroupsIDs, err := getSecurityGroupsIDs(machineProviderConfig.SecurityGroups, client) if err != nil { - return nil, fmt.Errorf("error getting security groups IDs: %v,", err) + return nil, mapierrors.InvalidMachineConfiguration("error getting security groups IDs: %v", err) } subnetIDs, err := getSubnetIDs(machineProviderConfig.Subnet, machineProviderConfig.Placement.AvailabilityZone, client) if err != nil { - return nil, fmt.Errorf("error getting subnet IDs: %v,", err) + return nil, mapierrors.InvalidMachineConfiguration("error getting subnet IDs: %v", err) } if len(subnetIDs) > 1 { glog.Warningf("More than one subnet id returned, only first one will be used") @@ -246,13 +249,13 @@ func launchInstance(machine *machinev1.Machine, machineProviderConfig *providerc blockDeviceMappings, err := getBlockDeviceMappings(machineProviderConfig.BlockDevices, *amiID, client) if err != nil { - return nil, fmt.Errorf("error getting blockDeviceMappings: %v,", err) + return nil, mapierrors.InvalidMachineConfiguration("error getting blockDeviceMappings: %v", err) } clusterID, ok := getClusterID(machine) if !ok { glog.Errorf("Unable to get cluster ID for machine: %q", machine.Name) - return nil, err + return nil, mapierrors.InvalidMachineConfiguration("Unable to get cluster ID for machine: %q", machine.Name) } // Add tags to the created machine rawTagList := []*ec2.Tag{} @@ -308,13 +311,25 @@ func launchInstance(machine *machinev1.Machine, machineProviderConfig *providerc } runResult, err := client.RunInstances(&inputConfig) if err != nil { + // we return InvalidMachineConfiguration for 4xx errors which by convention signal client misconfiguration + // https://tools.ietf.org/html/rfc2616#section-6.1.1 + // https: //docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html + // https://docs.aws.amazon.com/sdk-for-go/api/aws/awserr/ + if _, ok := err.(awserr.Error); ok { + if reqErr, ok := err.(awserr.RequestFailure); ok { + if strings.HasPrefix(strconv.Itoa(reqErr.StatusCode()), "4") { + glog.Infof("Error launching instance: %v", reqErr) + return nil, mapierrors.InvalidMachineConfiguration("error launching instance: %v", reqErr.Message()) + } + } + } glog.Errorf("Error creating EC2 instance: %v", err) - return nil, fmt.Errorf("error creating EC2 instance: %v", err) + return nil, mapierrors.CreateMachine("error creating EC2 instance: %v", err) } if runResult == nil || len(runResult.Instances) != 1 { glog.Errorf("Unexpected reservation creating instances: %v", runResult) - return nil, fmt.Errorf("unexpected reservation creating instance") + return nil, mapierrors.CreateMachine("unexpected reservation creating instance") } return runResult.Instances[0], nil diff --git a/pkg/client/client.go b/pkg/client/client.go index 2d3ac7c75b..e55504a4d7 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -18,7 +18,6 @@ package client import ( "context" - "fmt" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/cluster-api-provider-aws/pkg/version" @@ -34,6 +33,8 @@ import ( "github.com/aws/aws-sdk-go/service/elb/elbiface" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" + machineapiapierrors "github.com/openshift/cluster-api/pkg/errors" + apimachineryerrors "k8s.io/apimachinery/pkg/api/errors" ) //go:generate go run ../../vendor/github.com/golang/mock/mockgen -source=./client.go -destination=./mock/client_generated.go -package=mock @@ -134,16 +135,19 @@ func NewClient(ctrlRuntimeClient client.Client, secretName, namespace, region st if secretName != "" { var secret corev1.Secret if err := ctrlRuntimeClient.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: secretName}, &secret); err != nil { + if apimachineryerrors.IsNotFound(err) { + return nil, machineapiapierrors.InvalidMachineConfiguration("aws credentials secret %s/%s: %v not found", namespace, secretName, err) + } return nil, err } accessKeyID, ok := secret.Data[AwsCredsSecretIDKey] if !ok { - return nil, fmt.Errorf("AWS credentials secret %v did not contain key %v", + return nil, machineapiapierrors.InvalidMachineConfiguration("AWS credentials secret %v did not contain key %v", secretName, AwsCredsSecretIDKey) } secretAccessKey, ok := secret.Data[AwsCredsSecretAccessKey] if !ok { - return nil, fmt.Errorf("AWS credentials secret %v did not contain key %v", + return nil, machineapiapierrors.InvalidMachineConfiguration("AWS credentials secret %v did not contain key %v", secretName, AwsCredsSecretAccessKey) } From d26fda83e9e99c430ed90da107bf590a3ecab161 Mon Sep 17 00:00:00 2001 From: Brad Ison Date: Thu, 24 Oct 2019 12:41:23 +0200 Subject: [PATCH 02/14] Fix typo in file name --- pkg/actuators/machine/{instaces_test.go => instances_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/actuators/machine/{instaces_test.go => instances_test.go} (100%) diff --git a/pkg/actuators/machine/instaces_test.go b/pkg/actuators/machine/instances_test.go similarity index 100% rename from pkg/actuators/machine/instaces_test.go rename to pkg/actuators/machine/instances_test.go From 8d08b05f4cb22912dd615b3ad3358423c0ed418e Mon Sep 17 00:00:00 2001 From: Brad Ison Date: Wed, 23 Oct 2019 16:24:25 +0200 Subject: [PATCH 03/14] Search for existing instances by instance ID This changes the behavior of the actuator's `getMachineInstances` method to first look for a machine's backing instance by the instance ID in the provider status. If the instance ID is not set, it falls back to the previous method of filtering by tags. This should help prevent issues where AWS creates the instance, but takes some time to propagate tag values, which can result in Exists() wrongly returning false. --- pkg/actuators/machine/actuator.go | 18 + pkg/actuators/machine/actuator_test.go | 461 +++++++++++++++++++++++++ pkg/actuators/machine/utils.go | 64 +++- 3 files changed, 536 insertions(+), 7 deletions(-) diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index e097f5d6e1..9c1c7d39e5 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -541,6 +541,24 @@ func (a *Actuator) getMachineInstances(cluster *clusterv1.Cluster, machine *mach return nil, err } + status := &providerconfigv1.AWSMachineProviderStatus{} + err = a.codec.DecodeProviderStatus(machine.Status.ProviderStatus, status) + + // If the status was decoded successfully, and there is a non-empty instance + // ID, search using that, otherwise fallback to filtering based on tags. + if err == nil && status.InstanceID != nil && *status.InstanceID != "" { + i, err := getExistingInstanceByID(*status.InstanceID, client) + + if err != nil { + glog.Warningf("%s: Failed to find running instance by id %s: %v", + machine.Name, *status.InstanceID, err) + } else { + glog.Infof("%s: Found instance by id: %s", machine.Name, *status.InstanceID) + + return []*ec2.Instance{i}, nil + } + } + return getExistingInstances(machine, client) } diff --git a/pkg/actuators/machine/actuator_test.go b/pkg/actuators/machine/actuator_test.go index 4d7244a76b..b1d34128c8 100644 --- a/pkg/actuators/machine/actuator_test.go +++ b/pkg/actuators/machine/actuator_test.go @@ -16,10 +16,12 @@ import ( machineapierrors "github.com/openshift/cluster-api/pkg/errors" "github.com/stretchr/testify/assert" apiv1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1" awsclient "sigs.k8s.io/cluster-api-provider-aws/pkg/client" mockaws "sigs.k8s.io/cluster-api-provider-aws/pkg/client/mock" @@ -811,3 +813,462 @@ func TestGetUserData(t *testing.T) { } } } + +func TestCreate(t *testing.T) { + // mock aws API calls + mockCtrl := gomock.NewController(t) + mockAWSClient := mockaws.NewMockClient(mockCtrl) + mockAWSClient.EXPECT().DescribeSecurityGroups(gomock.Any()).Return(nil, fmt.Errorf("describeSecurityGroups error")).AnyTimes() + mockAWSClient.EXPECT().DescribeAvailabilityZones(gomock.Any()).Return(nil, fmt.Errorf("describeAvailabilityZones error")).AnyTimes() + mockAWSClient.EXPECT().DescribeImages(gomock.Any()).Return(nil, fmt.Errorf("describeImages error")).AnyTimes() + mockAWSClient.EXPECT().DescribeInstances(gomock.Any()).Return(stubDescribeInstancesOutput("ami-a9acbbd6", "i-02fcb933c5da7085c"), nil).AnyTimes() + mockAWSClient.EXPECT().TerminateInstances(gomock.Any()).Return(&ec2.TerminateInstancesOutput{}, nil).AnyTimes() + mockAWSClient.EXPECT().RunInstances(gomock.Any()).Return(stubReservation("ami-a9acbbd6", "i-02fcb933c5da7085c"), nil).AnyTimes() + mockAWSClient.EXPECT().RegisterInstancesWithLoadBalancer(gomock.Any()).Return(nil, nil).AnyTimes() + mockAWSClient.EXPECT().ELBv2DescribeLoadBalancers(gomock.Any()).Return(stubDescribeLoadBalancersOutput(), nil) + mockAWSClient.EXPECT().ELBv2DescribeTargetGroups(gomock.Any()).Return(stubDescribeTargetGroupsOutput(), nil).AnyTimes() + mockAWSClient.EXPECT().ELBv2RegisterTargets(gomock.Any()).Return(nil, nil).AnyTimes() + + testCases := []struct { + testcase string + providerConfig *providerconfigv1.AWSMachineProviderConfig + userDataSecret *corev1.Secret + awsCredentialsSecret *corev1.Secret + error error + }{ + { + testcase: "Create succeed", + providerConfig: &providerconfigv1.AWSMachineProviderConfig{ + AMI: providerconfigv1.AWSResourceReference{ + ID: aws.String("ami-a9acbbd6"), + }, + CredentialsSecret: &corev1.LocalObjectReference{ + Name: awsCredentialsSecretName, + }, + InstanceType: "m4.xlarge", + Placement: providerconfigv1.Placement{ + Region: region, + AvailabilityZone: defaultAvailabilityZone, + }, + Subnet: providerconfigv1.AWSResourceReference{ + ID: aws.String("subnet-0e56b13a64ff8a941"), + }, + IAMInstanceProfile: &providerconfigv1.AWSResourceReference{ + ID: aws.String("openshift_master_launch_instances"), + }, + KeyName: aws.String(keyName), + UserDataSecret: &corev1.LocalObjectReference{ + Name: userDataSecretName, + }, + Tags: []providerconfigv1.TagSpecification{ + {Name: "openshift-node-group-config", Value: "node-config-master"}, + {Name: "host-type", Value: "master"}, + {Name: "sub-host-type", Value: "default"}, + }, + SecurityGroups: []providerconfigv1.AWSResourceReference{ + {ID: aws.String("sg-00868b02fbe29de17")}, + {ID: aws.String("sg-0a4658991dc5eb40a")}, + {ID: aws.String("sg-009a70e28fa4ba84e")}, + {ID: aws.String("sg-07323d56fb932c84c")}, + {ID: aws.String("sg-08b1ffd32874d59a2")}, + }, + PublicIP: aws.Bool(true), + LoadBalancers: []providerconfigv1.LoadBalancerReference{ + { + Name: "cluster-con", + Type: providerconfigv1.ClassicLoadBalancerType, + }, + { + Name: "cluster-ext", + Type: providerconfigv1.ClassicLoadBalancerType, + }, + { + Name: "cluster-int", + Type: providerconfigv1.ClassicLoadBalancerType, + }, + { + Name: "cluster-net-lb", + Type: providerconfigv1.NetworkLoadBalancerType, + }, + }, + }, + userDataSecret: stubUserDataSecret(), + awsCredentialsSecret: stubAwsCredentialsSecret(), + error: nil, + }, + { + testcase: "Bad userData", + providerConfig: &providerconfigv1.AWSMachineProviderConfig{ + AMI: providerconfigv1.AWSResourceReference{ + ID: aws.String("ami-a9acbbd6"), + }, + CredentialsSecret: &corev1.LocalObjectReference{ + Name: awsCredentialsSecretName, + }, + InstanceType: "m4.xlarge", + Placement: providerconfigv1.Placement{ + Region: region, + AvailabilityZone: defaultAvailabilityZone, + }, + Subnet: providerconfigv1.AWSResourceReference{ + ID: aws.String("subnet-0e56b13a64ff8a941"), + }, + IAMInstanceProfile: &providerconfigv1.AWSResourceReference{ + ID: aws.String("openshift_master_launch_instances"), + }, + KeyName: aws.String(keyName), + UserDataSecret: &corev1.LocalObjectReference{ + Name: userDataSecretName, + }, + Tags: []providerconfigv1.TagSpecification{ + {Name: "openshift-node-group-config", Value: "node-config-master"}, + {Name: "host-type", Value: "master"}, + {Name: "sub-host-type", Value: "default"}, + }, + SecurityGroups: []providerconfigv1.AWSResourceReference{ + {ID: aws.String("sg-00868b02fbe29de17")}, + {ID: aws.String("sg-0a4658991dc5eb40a")}, + {ID: aws.String("sg-009a70e28fa4ba84e")}, + {ID: aws.String("sg-07323d56fb932c84c")}, + {ID: aws.String("sg-08b1ffd32874d59a2")}, + }, + PublicIP: aws.Bool(true), + LoadBalancers: []providerconfigv1.LoadBalancerReference{ + { + Name: "cluster-con", + Type: providerconfigv1.ClassicLoadBalancerType, + }, + { + Name: "cluster-ext", + Type: providerconfigv1.ClassicLoadBalancerType, + }, + { + Name: "cluster-int", + Type: providerconfigv1.ClassicLoadBalancerType, + }, + { + Name: "cluster-net-lb", + Type: providerconfigv1.NetworkLoadBalancerType, + }, + }, + }, + userDataSecret: &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: userDataSecretName, + Namespace: defaultNamespace, + }, + Data: map[string][]byte{ + "badKey": []byte(userDataBlob), + }, + }, + awsCredentialsSecret: stubAwsCredentialsSecret(), + error: &machineapierrors.MachineError{}, + }, + { + testcase: "Failed security groups return invalid config machine error", + providerConfig: &providerconfigv1.AWSMachineProviderConfig{ + AMI: providerconfigv1.AWSResourceReference{ + ID: aws.String("ami-a9acbbd6"), + }, + CredentialsSecret: &corev1.LocalObjectReference{ + Name: awsCredentialsSecretName, + }, + InstanceType: "m4.xlarge", + Placement: providerconfigv1.Placement{ + Region: region, + AvailabilityZone: defaultAvailabilityZone, + }, + Subnet: providerconfigv1.AWSResourceReference{ + ID: aws.String("subnet-0e56b13a64ff8a941"), + }, + IAMInstanceProfile: &providerconfigv1.AWSResourceReference{ + ID: aws.String("openshift_master_launch_instances"), + }, + KeyName: aws.String(keyName), + UserDataSecret: &corev1.LocalObjectReference{ + Name: userDataSecretName, + }, + Tags: []providerconfigv1.TagSpecification{ + {Name: "openshift-node-group-config", Value: "node-config-master"}, + {Name: "host-type", Value: "master"}, + {Name: "sub-host-type", Value: "default"}, + }, + SecurityGroups: []providerconfigv1.AWSResourceReference{{ + Filters: []providerconfigv1.Filter{{ + Name: "tag:Name", + Values: []string{fmt.Sprintf("%s-%s-sg", clusterID, "role")}, + }}, + }}, + PublicIP: aws.Bool(true), + }, + userDataSecret: stubUserDataSecret(), + awsCredentialsSecret: stubAwsCredentialsSecret(), + error: &machineapierrors.MachineError{}, + }, + { + testcase: "Failed Availability zones return invalid config machine error", + providerConfig: &providerconfigv1.AWSMachineProviderConfig{ + AMI: providerconfigv1.AWSResourceReference{ + ID: aws.String("ami-a9acbbd6"), + }, + CredentialsSecret: &corev1.LocalObjectReference{ + Name: awsCredentialsSecretName, + }, + InstanceType: "m4.xlarge", + Placement: providerconfigv1.Placement{ + Region: region, + AvailabilityZone: defaultAvailabilityZone, + }, + Subnet: providerconfigv1.AWSResourceReference{ + Filters: []providerconfigv1.Filter{{ + Name: "tag:Name", + Values: []string{fmt.Sprintf("%s-private-%s", clusterID, "az")}, + }}, + }, + IAMInstanceProfile: &providerconfigv1.AWSResourceReference{ + ID: aws.String("openshift_master_launch_instances"), + }, + KeyName: aws.String(keyName), + UserDataSecret: &corev1.LocalObjectReference{ + Name: userDataSecretName, + }, + Tags: []providerconfigv1.TagSpecification{ + {Name: "openshift-node-group-config", Value: "node-config-master"}, + {Name: "host-type", Value: "master"}, + {Name: "sub-host-type", Value: "default"}, + }, + SecurityGroups: []providerconfigv1.AWSResourceReference{ + {ID: aws.String("sg-00868b02fbe29de17")}, + {ID: aws.String("sg-0a4658991dc5eb40a")}, + {ID: aws.String("sg-009a70e28fa4ba84e")}, + {ID: aws.String("sg-07323d56fb932c84c")}, + {ID: aws.String("sg-08b1ffd32874d59a2")}, + }, + PublicIP: aws.Bool(true), + }, + userDataSecret: stubUserDataSecret(), + awsCredentialsSecret: stubAwsCredentialsSecret(), + error: &machineapierrors.MachineError{}, + }, + { + testcase: "Failed BlockDevices return invalid config machine error", + providerConfig: &providerconfigv1.AWSMachineProviderConfig{ + AMI: providerconfigv1.AWSResourceReference{ + ID: aws.String("ami-a9acbbd6"), + }, + CredentialsSecret: &corev1.LocalObjectReference{ + Name: awsCredentialsSecretName, + }, + InstanceType: "m4.xlarge", + Placement: providerconfigv1.Placement{ + Region: region, + AvailabilityZone: defaultAvailabilityZone, + }, + BlockDevices: []providerconfigv1.BlockDeviceMappingSpec{ + { + EBS: &providerconfigv1.EBSBlockDeviceSpec{ + VolumeType: pointer.StringPtr("type"), + VolumeSize: pointer.Int64Ptr(int64(1)), + Iops: pointer.Int64Ptr(int64(1)), + }, + }, + }, + Subnet: providerconfigv1.AWSResourceReference{ + ID: aws.String("subnet-0e56b13a64ff8a941"), + }, + IAMInstanceProfile: &providerconfigv1.AWSResourceReference{ + ID: aws.String("openshift_master_launch_instances"), + }, + KeyName: aws.String(keyName), + UserDataSecret: &corev1.LocalObjectReference{ + Name: userDataSecretName, + }, + Tags: []providerconfigv1.TagSpecification{ + {Name: "openshift-node-group-config", Value: "node-config-master"}, + {Name: "host-type", Value: "master"}, + {Name: "sub-host-type", Value: "default"}, + }, + SecurityGroups: []providerconfigv1.AWSResourceReference{ + {ID: aws.String("sg-00868b02fbe29de17")}, + {ID: aws.String("sg-0a4658991dc5eb40a")}, + {ID: aws.String("sg-009a70e28fa4ba84e")}, + {ID: aws.String("sg-07323d56fb932c84c")}, + {ID: aws.String("sg-08b1ffd32874d59a2")}, + }, + PublicIP: aws.Bool(true), + }, + userDataSecret: stubUserDataSecret(), + awsCredentialsSecret: stubAwsCredentialsSecret(), + error: &machineapierrors.MachineError{}, + }, + } + + for _, tc := range testCases { + // create fake resources + t.Logf("testCase: %v", tc.testcase) + codec, err := providerconfigv1.NewCodec() + if err != nil { + t.Fatalf("unable to build codec: %v", err) + } + encodedProviderConfig, err := codec.EncodeProviderSpec(tc.providerConfig) + if err != nil { + t.Fatalf("Unexpected error") + } + machine, err := stubMachine() + if err != nil { + t.Fatal(err) + } + machine.Spec.ProviderSpec = *encodedProviderConfig + fakeClient := fake.NewFakeClientWithScheme(scheme.Scheme, machine, tc.awsCredentialsSecret, tc.userDataSecret) + + // create actuator + params := ActuatorParams{ + Client: fakeClient, + AwsClientBuilder: func(client client.Client, secretName, namespace, region string) (awsclient.Client, error) { + return mockAWSClient, nil + }, + Codec: codec, + // use empty recorder dropping any event recorded + EventRecorder: &record.FakeRecorder{}, + } + actuator, err := NewActuator(params) + if err != nil { + t.Fatalf("Could not create AWS machine actuator: %v", err) + } + + // test create + err = actuator.Create(context.TODO(), nil, machine) + if tc.error != nil { + if err == nil { + t.Fatalf("Expected error") + } + _, expectMachineError := tc.error.(*machineapierrors.MachineError) + _, gotMachineError := err.(*machineapierrors.MachineError) + if expectMachineError && !gotMachineError || !expectMachineError && gotMachineError { + t.Fatalf("Expected %T, got: %T", tc.error, err) + } + } else if err != nil { + t.Fatalf("Unexpected error") + } + } +} + +func TestGetMachineInstances(t *testing.T) { + clusterID := "aws-actuator-cluster" + instanceID := "i-02fa4197109214b46" + imageID := "ami-a9acbbd6" + + machine, err := stubMachine() + if err != nil { + t.Fatalf("unable to build stub machine: %v", err) + } + + codec, err := providerconfigv1.NewCodec() + if err != nil { + t.Fatalf("unable to build codec: %v", err) + } + + testCases := []struct { + testcase string + providerStatus providerconfigv1.AWSMachineProviderStatus + awsClientFunc func(*gomock.Controller) awsclient.Client + }{ + { + testcase: "empty-status-search-by-tag", + providerStatus: providerconfigv1.AWSMachineProviderStatus{}, + awsClientFunc: func(ctrl *gomock.Controller) awsclient.Client { + mockAWSClient := mockaws.NewMockClient(ctrl) + + request := &ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + { + Name: awsTagFilter("Name"), + Values: aws.StringSlice([]string{machine.Name}), + }, + + clusterFilter(clusterID), + + { + Name: aws.String("instance-state-name"), + Values: existingInstanceStates(), + }, + }, + } + + mockAWSClient.EXPECT().DescribeInstances(request).Return( + stubDescribeInstancesOutput(imageID, instanceID), + nil, + ).Times(1) + + return mockAWSClient + }, + }, + { + testcase: "has-status-search-by-id", + providerStatus: providerconfigv1.AWSMachineProviderStatus{ + InstanceID: aws.String(instanceID), + }, + awsClientFunc: func(ctrl *gomock.Controller) awsclient.Client { + mockAWSClient := mockaws.NewMockClient(ctrl) + + request := &ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("instance-id"), + Values: aws.StringSlice([]string{instanceID}), + }, + { + Name: aws.String("instance-state-name"), + Values: existingInstanceStates(), + }, + }, + } + + mockAWSClient.EXPECT().DescribeInstances(request).Return( + stubDescribeInstancesOutput(imageID, instanceID), + nil, + ).Times(1) + + return mockAWSClient + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.testcase, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + awsStatusRaw, err := codec.EncodeProviderStatus(&tc.providerStatus) + if err != nil { + t.Errorf("Error encoding ProviderStatus: %v", err) + } + + machineCopy := machine.DeepCopy() + machineCopy.Status.ProviderStatus = awsStatusRaw + + awsClient := tc.awsClientFunc(ctrl) + + params := ActuatorParams{ + Codec: codec, + AwsClientBuilder: awsClientBuilderFunc(awsClient), + } + + actuator, err := NewActuator(params) + if err != nil { + t.Errorf("Error creating Actuator: %v", err) + } + + _, err = actuator.getMachineInstances(nil, machineCopy) + if err != nil { + t.Errorf("Unexpected error from getMachineInstances: %v", err) + } + }) + } +} + +func awsClientBuilderFunc(c awsclient.Client) awsclient.AwsClientBuilderFuncType { + return func(_ client.Client, _, _, _ string) (awsclient.Client, error) { + return c, nil + } +} diff --git a/pkg/actuators/machine/utils.go b/pkg/actuators/machine/utils.go index d5e19c6d8f..cb088b429b 100644 --- a/pkg/actuators/machine/utils.go +++ b/pkg/actuators/machine/utils.go @@ -34,6 +34,18 @@ import ( awsclient "sigs.k8s.io/cluster-api-provider-aws/pkg/client" ) +// existingInstanceStates returns the list of states an EC2 instance can be in +// while being considered "existing", i.e. mostly anything but "Terminated". +func existingInstanceStates() []*string { + return []*string{ + aws.String(ec2.InstanceStateNameRunning), + aws.String(ec2.InstanceStateNamePending), + aws.String(ec2.InstanceStateNameStopped), + aws.String(ec2.InstanceStateNameStopping), + aws.String(ec2.InstanceStateNameShuttingDown), + } +} + // getRunningFromInstances returns all running instances from a list of instances. func getRunningFromInstances(instances []*ec2.Instance) []*ec2.Instance { var runningInstances []*ec2.Instance @@ -53,13 +65,51 @@ func getStoppedInstances(machine *machinev1.Machine, client awsclient.Client) ([ } func getExistingInstances(machine *machinev1.Machine, client awsclient.Client) ([]*ec2.Instance, error) { - return getInstances(machine, client, []*string{ - aws.String(ec2.InstanceStateNameRunning), - aws.String(ec2.InstanceStateNamePending), - aws.String(ec2.InstanceStateNameStopped), - aws.String(ec2.InstanceStateNameStopping), - aws.String(ec2.InstanceStateNameShuttingDown), - }) + return getInstances(machine, client, existingInstanceStates()) +} + +func getExistingInstanceByID(id string, client awsclient.Client) (*ec2.Instance, error) { + return getInstanceByID(id, client, existingInstanceStates()) +} + +// getInstanceByID returns the instance with the given ID if it exists. +func getInstanceByID(id string, client awsclient.Client, instanceStateFilter []*string) (*ec2.Instance, error) { + if id == "" { + return nil, fmt.Errorf("instance-id not specified") + } + + requestFilters := []*ec2.Filter{ + { + Name: aws.String("instance-id"), + Values: aws.StringSlice([]string{id}), + }, + } + + if instanceStateFilter != nil { + requestFilters = append(requestFilters, &ec2.Filter{ + Name: aws.String("instance-state-name"), + Values: instanceStateFilter, + }) + } + + request := &ec2.DescribeInstancesInput{Filters: requestFilters} + + result, err := client.DescribeInstances(request) + if err != nil { + return nil, err + } + + if len(result.Reservations) != 1 { + return nil, fmt.Errorf("found %d reservations for instance-id %s", len(result.Reservations), id) + } + + reservation := result.Reservations[0] + + if len(reservation.Instances) != 1 { + return nil, fmt.Errorf("found %d instances for instance-id %s", len(reservation.Instances), id) + } + + return reservation.Instances[0], nil } // getInstances returns all instances that have a tag matching our machine name, From 744805b668eed1730873e1b1268eab99d0a3a53f Mon Sep 17 00:00:00 2001 From: Enxebre Date: Thu, 14 Nov 2019 12:38:34 +0100 Subject: [PATCH 04/14] Stop retieving instances by state on API call This attemps to mitigate https://bugzilla.redhat.com/show_bug.cgi?id=1772163. Instead of telling the API request to filter by state, we fetch all and discriminate only terminated instances --- pkg/actuators/machine/actuator.go | 10 ++--- pkg/actuators/machine/actuator_test.go | 57 +++++++++++++++++++------- pkg/actuators/machine/stubs.go | 20 +++++++++ pkg/actuators/machine/utils.go | 47 ++++++++++++--------- 4 files changed, 95 insertions(+), 39 deletions(-) diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index 9c1c7d39e5..6fe17af748 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -486,14 +486,14 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m return a.updateStatus(machine, newestInstance) } -// Exists determines if the given machine currently exists. For AWS we query for instances in -// running state, with a matching name tag, to determine a match. +// Exists determines if the given machine currently exists. +// A machine which is not terminated is considered as existing. func (a *Actuator) Exists(context context.Context, cluster *clusterv1.Cluster, machine *machinev1.Machine) (bool, error) { glog.Infof("%s: Checking if machine exists", machine.Name) instances, err := a.getMachineInstances(cluster, machine) if err != nil { - glog.Errorf("%s: Error getting running instances: %v", machine.Name, err) + glog.Errorf("%s: Error getting existing instances: %v", machine.Name, err) return false, err } if len(instances) == 0 { @@ -548,13 +548,11 @@ func (a *Actuator) getMachineInstances(cluster *clusterv1.Cluster, machine *mach // ID, search using that, otherwise fallback to filtering based on tags. if err == nil && status.InstanceID != nil && *status.InstanceID != "" { i, err := getExistingInstanceByID(*status.InstanceID, client) - if err != nil { - glog.Warningf("%s: Failed to find running instance by id %s: %v", + glog.Warningf("%s: Failed to find existing instance by id %s: %v", machine.Name, *status.InstanceID, err) } else { glog.Infof("%s: Found instance by id: %s", machine.Name, *status.InstanceID) - return []*ec2.Instance{i}, nil } } diff --git a/pkg/actuators/machine/actuator_test.go b/pkg/actuators/machine/actuator_test.go index b1d34128c8..09641e6ea5 100644 --- a/pkg/actuators/machine/actuator_test.go +++ b/pkg/actuators/machine/actuator_test.go @@ -1172,6 +1172,7 @@ func TestGetMachineInstances(t *testing.T) { testcase string providerStatus providerconfigv1.AWSMachineProviderStatus awsClientFunc func(*gomock.Controller) awsclient.Client + exists bool }{ { testcase: "empty-status-search-by-tag", @@ -1187,11 +1188,6 @@ func TestGetMachineInstances(t *testing.T) { }, clusterFilter(clusterID), - - { - Name: aws.String("instance-state-name"), - Values: existingInstanceStates(), - }, }, } @@ -1202,6 +1198,7 @@ func TestGetMachineInstances(t *testing.T) { return mockAWSClient }, + exists: true, }, { testcase: "has-status-search-by-id", @@ -1212,25 +1209,54 @@ func TestGetMachineInstances(t *testing.T) { mockAWSClient := mockaws.NewMockClient(ctrl) request := &ec2.DescribeInstancesInput{ + InstanceIds: aws.StringSlice([]string{instanceID}), + } + + mockAWSClient.EXPECT().DescribeInstances(request).Return( + stubDescribeInstancesOutput(imageID, instanceID), + nil, + ).Times(1) + + return mockAWSClient + }, + exists: true, + }, + { + testcase: "has-status-search-by-id and machine is terminated", + providerStatus: providerconfigv1.AWSMachineProviderStatus{ + InstanceID: aws.String(instanceID), + }, + awsClientFunc: func(ctrl *gomock.Controller) awsclient.Client { + mockAWSClient := mockaws.NewMockClient(ctrl) + + request := &ec2.DescribeInstancesInput{ + InstanceIds: aws.StringSlice([]string{instanceID}), + } + + mockAWSClient.EXPECT().DescribeInstances(request).Return( + stubTerminatedInstanceDescribeInstancesOutput(imageID, instanceID), + nil, + ).Times(1) + + request2 := &ec2.DescribeInstancesInput{ Filters: []*ec2.Filter{ { - Name: aws.String("instance-id"), - Values: aws.StringSlice([]string{instanceID}), - }, - { - Name: aws.String("instance-state-name"), - Values: existingInstanceStates(), + Name: awsTagFilter("Name"), + Values: aws.StringSlice([]string{machine.Name}), }, + + clusterFilter(clusterID), }, } - mockAWSClient.EXPECT().DescribeInstances(request).Return( - stubDescribeInstancesOutput(imageID, instanceID), + mockAWSClient.EXPECT().DescribeInstances(request2).Return( + stubTerminatedInstanceDescribeInstancesOutput(imageID, instanceID), nil, ).Times(1) return mockAWSClient }, + exists: false, }, } @@ -1259,10 +1285,13 @@ func TestGetMachineInstances(t *testing.T) { t.Errorf("Error creating Actuator: %v", err) } - _, err = actuator.getMachineInstances(nil, machineCopy) + instance, err := actuator.getMachineInstances(nil, machineCopy) if err != nil { t.Errorf("Unexpected error from getMachineInstances: %v", err) } + if tc.exists != (instance != nil) { + t.Errorf("Expected instance exists: %t, got instance: %v", tc.exists, instance) + } }) } } diff --git a/pkg/actuators/machine/stubs.go b/pkg/actuators/machine/stubs.go index d670cb070d..47234580e4 100644 --- a/pkg/actuators/machine/stubs.go +++ b/pkg/actuators/machine/stubs.go @@ -279,3 +279,23 @@ func stubDescribeInstancesOutput(imageID, instanceID string, state string) *ec2. }, } } + +func stubTerminatedInstanceDescribeInstancesOutput(imageID, instanceID string) *ec2.DescribeInstancesOutput { + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + { + ImageId: aws.String(imageID), + InstanceId: aws.String(instanceID), + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameTerminated), + Code: aws.Int64(16), + }, + LaunchTime: aws.Time(time.Now()), + }, + }, + }, + }, + } +} diff --git a/pkg/actuators/machine/utils.go b/pkg/actuators/machine/utils.go index cb088b429b..25ecd391ce 100644 --- a/pkg/actuators/machine/utils.go +++ b/pkg/actuators/machine/utils.go @@ -64,36 +64,47 @@ func getStoppedInstances(machine *machinev1.Machine, client awsclient.Client) ([ return getInstances(machine, client, stoppedInstanceStateFilter) } +// getExistingInstances returns all instances not terminated func getExistingInstances(machine *machinev1.Machine, client awsclient.Client) ([]*ec2.Instance, error) { - return getInstances(machine, client, existingInstanceStates()) + instances, err := getInstances(machine, client, nil) + if err != nil { + return nil, err + } + + var existingInstances []*ec2.Instance + for key := range instances { + if (instances[key].State) != nil { + if aws.StringValue(instances[key].State.Name) != ec2.InstanceStateNameTerminated { + existingInstances = append(existingInstances, instances[key]) + } + } + } + return existingInstances, nil } func getExistingInstanceByID(id string, client awsclient.Client) (*ec2.Instance, error) { - return getInstanceByID(id, client, existingInstanceStates()) + instance, err := getInstanceByID(id, client) + if err != nil { + return nil, err + } + if instance.State != nil { + if aws.StringValue(instance.State.Name) == ec2.InstanceStateNameTerminated { + return nil, fmt.Errorf("failed to getExistingInstanceByID for instance-id %s, instance is terminated", id) + } + } + return instance, nil } // getInstanceByID returns the instance with the given ID if it exists. -func getInstanceByID(id string, client awsclient.Client, instanceStateFilter []*string) (*ec2.Instance, error) { +func getInstanceByID(id string, client awsclient.Client) (*ec2.Instance, error) { if id == "" { return nil, fmt.Errorf("instance-id not specified") } - requestFilters := []*ec2.Filter{ - { - Name: aws.String("instance-id"), - Values: aws.StringSlice([]string{id}), - }, - } - - if instanceStateFilter != nil { - requestFilters = append(requestFilters, &ec2.Filter{ - Name: aws.String("instance-state-name"), - Values: instanceStateFilter, - }) + request := &ec2.DescribeInstancesInput{ + InstanceIds: aws.StringSlice([]string{id}), } - request := &ec2.DescribeInstancesInput{Filters: requestFilters} - result, err := client.DescribeInstances(request) if err != nil { return nil, err @@ -115,7 +126,6 @@ func getInstanceByID(id string, client awsclient.Client, instanceStateFilter []* // getInstances returns all instances that have a tag matching our machine name, // and cluster ID. func getInstances(machine *machinev1.Machine, client awsclient.Client, instanceStateFilter []*string) ([]*ec2.Instance, error) { - clusterID, ok := getClusterID(machine) if !ok { return []*ec2.Instance{}, fmt.Errorf("unable to get cluster ID for machine: %q", machine.Name) @@ -128,7 +138,6 @@ func getInstances(machine *machinev1.Machine, client awsclient.Client, instanceS }, clusterFilter(clusterID), } - if instanceStateFilter != nil { requestFilters = append(requestFilters, &ec2.Filter{ Name: aws.String("instance-state-name"), From 3038f30cec47413a7c5ffbb02a19c8e6de9a2fbc Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 13 Nov 2019 11:54:50 -0800 Subject: [PATCH 05/14] pkg/actuators/machine/utils: Drop running/pending comment from getInstances getInstances has taken an instanceStateFilter argument since it landed as GetInstances in 72b482323c (Migrate cluster operator implementation of the aws actuator, 2018-08-04, openshift/cluster-api-provider-aws#2). --- pkg/actuators/machine/utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/actuators/machine/utils.go b/pkg/actuators/machine/utils.go index 25ecd391ce..6d5d122cdb 100644 --- a/pkg/actuators/machine/utils.go +++ b/pkg/actuators/machine/utils.go @@ -145,7 +145,6 @@ func getInstances(machine *machinev1.Machine, client awsclient.Client, instanceS }) } - // Query instances with our machine's name, and in running/pending state. request := &ec2.DescribeInstancesInput{ Filters: requestFilters, } From e45debbed4ed103b85f4467494f7c644b72eb320 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 13 Nov 2019 20:56:00 -0800 Subject: [PATCH 06/14] pkg/actuators/machine/stubs: Add 'state' argument to stubDescribeInstancesOutput So we can create pending, terminated, etc. instances for testing. This should scale better than stub{State}InstanceDescribeInstancesOutput from openshift/cluster-api-provider-aws@c8e341f22f (Stop retieving instances by state on API call, 2019-11-14, openshift/cluster-api-provider-aws#269). --- pkg/actuators/machine/actuator_test.go | 14 +++++++------- pkg/actuators/machine/stubs.go | 20 -------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/pkg/actuators/machine/actuator_test.go b/pkg/actuators/machine/actuator_test.go index 09641e6ea5..15ff761e0f 100644 --- a/pkg/actuators/machine/actuator_test.go +++ b/pkg/actuators/machine/actuator_test.go @@ -673,7 +673,7 @@ func TestAvailabiltyZone(t *testing.T) { ImageId: aws.String("ami-a9acbbd6"), InstanceId: aws.String("i-02fcb933c5da7085c"), State: &ec2.InstanceState{ - Name: aws.String("Running"), + Name: aws.String(ec2.InstanceStateNameRunning), }, LaunchTime: aws.Time(time.Now()), Placement: &ec2.Placement{ @@ -692,7 +692,7 @@ func TestAvailabiltyZone(t *testing.T) { ImageId: aws.String("ami-a9acbbd6"), InstanceId: aws.String("i-02fcb933c5da7085c"), State: &ec2.InstanceState{ - Name: aws.String("Running"), + Name: aws.String(ec2.InstanceStateNameRunning), Code: aws.Int64(16), }, LaunchTime: aws.Time(time.Now()), @@ -821,7 +821,7 @@ func TestCreate(t *testing.T) { mockAWSClient.EXPECT().DescribeSecurityGroups(gomock.Any()).Return(nil, fmt.Errorf("describeSecurityGroups error")).AnyTimes() mockAWSClient.EXPECT().DescribeAvailabilityZones(gomock.Any()).Return(nil, fmt.Errorf("describeAvailabilityZones error")).AnyTimes() mockAWSClient.EXPECT().DescribeImages(gomock.Any()).Return(nil, fmt.Errorf("describeImages error")).AnyTimes() - mockAWSClient.EXPECT().DescribeInstances(gomock.Any()).Return(stubDescribeInstancesOutput("ami-a9acbbd6", "i-02fcb933c5da7085c"), nil).AnyTimes() + mockAWSClient.EXPECT().DescribeInstances(gomock.Any()).Return(stubDescribeInstancesOutput("ami-a9acbbd6", "i-02fcb933c5da7085c", ec2.InstanceStateNameRunning), nil).AnyTimes() mockAWSClient.EXPECT().TerminateInstances(gomock.Any()).Return(&ec2.TerminateInstancesOutput{}, nil).AnyTimes() mockAWSClient.EXPECT().RunInstances(gomock.Any()).Return(stubReservation("ami-a9acbbd6", "i-02fcb933c5da7085c"), nil).AnyTimes() mockAWSClient.EXPECT().RegisterInstancesWithLoadBalancer(gomock.Any()).Return(nil, nil).AnyTimes() @@ -1192,7 +1192,7 @@ func TestGetMachineInstances(t *testing.T) { } mockAWSClient.EXPECT().DescribeInstances(request).Return( - stubDescribeInstancesOutput(imageID, instanceID), + stubDescribeInstancesOutput(imageID, instanceID, ec2.InstanceStateNameRunning), nil, ).Times(1) @@ -1213,7 +1213,7 @@ func TestGetMachineInstances(t *testing.T) { } mockAWSClient.EXPECT().DescribeInstances(request).Return( - stubDescribeInstancesOutput(imageID, instanceID), + stubDescribeInstancesOutput(imageID, instanceID, ec2.InstanceStateNameRunning), nil, ).Times(1) @@ -1234,7 +1234,7 @@ func TestGetMachineInstances(t *testing.T) { } mockAWSClient.EXPECT().DescribeInstances(request).Return( - stubTerminatedInstanceDescribeInstancesOutput(imageID, instanceID), + stubDescribeInstancesOutput(imageID, instanceID, ec2.InstanceStateNameTerminated), nil, ).Times(1) @@ -1250,7 +1250,7 @@ func TestGetMachineInstances(t *testing.T) { } mockAWSClient.EXPECT().DescribeInstances(request2).Return( - stubTerminatedInstanceDescribeInstancesOutput(imageID, instanceID), + stubDescribeInstancesOutput(imageID, instanceID, ec2.InstanceStateNameTerminated), nil, ).Times(1) diff --git a/pkg/actuators/machine/stubs.go b/pkg/actuators/machine/stubs.go index 47234580e4..d670cb070d 100644 --- a/pkg/actuators/machine/stubs.go +++ b/pkg/actuators/machine/stubs.go @@ -279,23 +279,3 @@ func stubDescribeInstancesOutput(imageID, instanceID string, state string) *ec2. }, } } - -func stubTerminatedInstanceDescribeInstancesOutput(imageID, instanceID string) *ec2.DescribeInstancesOutput { - return &ec2.DescribeInstancesOutput{ - Reservations: []*ec2.Reservation{ - { - Instances: []*ec2.Instance{ - { - ImageId: aws.String(imageID), - InstanceId: aws.String(instanceID), - State: &ec2.InstanceState{ - Name: aws.String(ec2.InstanceStateNameTerminated), - Code: aws.Int64(16), - }, - LaunchTime: aws.Time(time.Now()), - }, - }, - }, - }, - } -} From ebace7f1726d7c7671b4c4f3f6971eb25f69bde4 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 13 Nov 2019 12:13:01 -0800 Subject: [PATCH 07/14] pkg/actuators/machine/utils: Client-side state filtering in getInstanceByID openshift/cluster-api-provider-aws@c8e341f22f (Stop retieving instances by state on API call, 2019-11-14, openshift/cluster-api-provider-aws#269) did part of this. This commit: * Restores the instanceStateFilter argument to getInstanceByID, to avoid having two definitions (existingInstanceStates and a local != Terminated in getExistingInstanceByID) that define what it means to exist. This will also give us logged error messages like: instance i-123 state terminated is not in running, pending, stopped, stopping, shutting-down which will make it easier to find and fix omissions in existingInstanceStates. * Adds a -running suffix to has-status-search-by-id, so it's easier to distinguish from the terminated case. * Converts to the hyphenated has-status-search-by-id-terminated, instead of mixing hyphens and spaces in a case name. * Uses inline argument for the DescribeInstances mocks, so we don't have to use request and request2. These aren't so big that inlining them makes the mock setup unreadable. * Returns an empty set of instances in the instance-listing fallback call in has-status-search-by-id-terminated (which we now require to happen after the by-ID call). The list call is filtered by state, so it wouldn't return a terminated instance. --- pkg/actuators/machine/actuator_test.go | 21 +++++------- pkg/actuators/machine/utils.go | 45 ++++++++++++++++---------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/pkg/actuators/machine/actuator_test.go b/pkg/actuators/machine/actuator_test.go index 15ff761e0f..07b355adac 100644 --- a/pkg/actuators/machine/actuator_test.go +++ b/pkg/actuators/machine/actuator_test.go @@ -1201,7 +1201,7 @@ func TestGetMachineInstances(t *testing.T) { exists: true, }, { - testcase: "has-status-search-by-id", + testcase: "has-status-search-by-id-running", providerStatus: providerconfigv1.AWSMachineProviderStatus{ InstanceID: aws.String(instanceID), }, @@ -1222,23 +1222,21 @@ func TestGetMachineInstances(t *testing.T) { exists: true, }, { - testcase: "has-status-search-by-id and machine is terminated", + testcase: "has-status-search-by-id-terminated", providerStatus: providerconfigv1.AWSMachineProviderStatus{ InstanceID: aws.String(instanceID), }, awsClientFunc: func(ctrl *gomock.Controller) awsclient.Client { mockAWSClient := mockaws.NewMockClient(ctrl) - request := &ec2.DescribeInstancesInput{ + first := mockAWSClient.EXPECT().DescribeInstances(&ec2.DescribeInstancesInput{ InstanceIds: aws.StringSlice([]string{instanceID}), - } - - mockAWSClient.EXPECT().DescribeInstances(request).Return( + }).Return( stubDescribeInstancesOutput(imageID, instanceID, ec2.InstanceStateNameTerminated), nil, ).Times(1) - request2 := &ec2.DescribeInstancesInput{ + mockAWSClient.EXPECT().DescribeInstances(&ec2.DescribeInstancesInput{ Filters: []*ec2.Filter{ { Name: awsTagFilter("Name"), @@ -1247,16 +1245,13 @@ func TestGetMachineInstances(t *testing.T) { clusterFilter(clusterID), }, - } - - mockAWSClient.EXPECT().DescribeInstances(request2).Return( - stubDescribeInstancesOutput(imageID, instanceID, ec2.InstanceStateNameTerminated), + }).Return( + &ec2.DescribeInstancesOutput{}, nil, - ).Times(1) + ).Times(1).After(first) return mockAWSClient }, - exists: false, }, } diff --git a/pkg/actuators/machine/utils.go b/pkg/actuators/machine/utils.go index 6d5d122cdb..4d7090d8b2 100644 --- a/pkg/actuators/machine/utils.go +++ b/pkg/actuators/machine/utils.go @@ -19,16 +19,15 @@ package machine import ( "fmt" "net" - - "github.com/golang/glog" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/golang/glog" machinev1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" "golang.org/x/net/context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1" awsclient "sigs.k8s.io/cluster-api-provider-aws/pkg/client" @@ -83,20 +82,11 @@ func getExistingInstances(machine *machinev1.Machine, client awsclient.Client) ( } func getExistingInstanceByID(id string, client awsclient.Client) (*ec2.Instance, error) { - instance, err := getInstanceByID(id, client) - if err != nil { - return nil, err - } - if instance.State != nil { - if aws.StringValue(instance.State.Name) == ec2.InstanceStateNameTerminated { - return nil, fmt.Errorf("failed to getExistingInstanceByID for instance-id %s, instance is terminated", id) - } - } - return instance, nil + return getInstanceByID(id, client, existingInstanceStates()) } // getInstanceByID returns the instance with the given ID if it exists. -func getInstanceByID(id string, client awsclient.Client) (*ec2.Instance, error) { +func getInstanceByID(id string, client awsclient.Client, instanceStateFilter []*string) (*ec2.Instance, error) { if id == "" { return nil, fmt.Errorf("instance-id not specified") } @@ -120,7 +110,28 @@ func getInstanceByID(id string, client awsclient.Client) (*ec2.Instance, error) return nil, fmt.Errorf("found %d instances for instance-id %s", len(reservation.Instances), id) } - return reservation.Instances[0], nil + instance := reservation.Instances[0] + + if len(instanceStateFilter) == 0 { + return instance, nil + } + + if instance.State == nil { + return nil, fmt.Errorf("instance %s has nil state", id) + } + + actualState := aws.StringValue(instance.State.Name) + for _, allowedState := range instanceStateFilter { + if aws.StringValue(allowedState) == actualState { + return instance, nil + } + } + + allowedStates := make([]string, 0, len(instanceStateFilter)) + for _, allowedState := range instanceStateFilter { + allowedStates = append(allowedStates, aws.StringValue(allowedState)) + } + return instance, fmt.Errorf("instance %s state %q is not in %s", id, actualState, strings.Join(allowedStates, ", ")) } // getInstances returns all instances that have a tag matching our machine name, From 473289a748140f44a419f9e35c5c168b093d9cf6 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 14 Nov 2019 09:35:57 -0800 Subject: [PATCH 08/14] pkg/actuators/machine/utils: Client-side state filtering in getInstances Like openshift/cluster-api-provider-aws@e4e94b1e08 (pkg/actuators/machine/utils: Client-side state filtering in getInstanceByID, 2019-11-13, openshift/cluster-api-provider-aws#268), but for getInstances. The cluster/name filtering is already enough to limit the list response to a reasonable size, so we can perform the state filtering locally in order to produce useful error messages. Also fix a bug in TestGetMachineInstances about the expected return type of getMachineInstances; it's a slice, not a single instance. Also, fix a few "Running" -> InstanceStateNameRunning bugs; the AWS string is "running", but we shouldn't care either way. --- pkg/actuators/machine/actuator_test.go | 8 +-- pkg/actuators/machine/stubs.go | 4 +- pkg/actuators/machine/utils.go | 75 ++++++++++++-------------- pkg/client/fake/fake.go | 2 +- 4 files changed, 42 insertions(+), 47 deletions(-) diff --git a/pkg/actuators/machine/actuator_test.go b/pkg/actuators/machine/actuator_test.go index 07b355adac..73af82ad8e 100644 --- a/pkg/actuators/machine/actuator_test.go +++ b/pkg/actuators/machine/actuator_test.go @@ -1246,7 +1246,7 @@ func TestGetMachineInstances(t *testing.T) { clusterFilter(clusterID), }, }).Return( - &ec2.DescribeInstancesOutput{}, + stubDescribeInstancesOutput(imageID, instanceID, ec2.InstanceStateNameTerminated), nil, ).Times(1).After(first) @@ -1280,12 +1280,12 @@ func TestGetMachineInstances(t *testing.T) { t.Errorf("Error creating Actuator: %v", err) } - instance, err := actuator.getMachineInstances(nil, machineCopy) + instances, err := actuator.getMachineInstances(nil, machineCopy) if err != nil { t.Errorf("Unexpected error from getMachineInstances: %v", err) } - if tc.exists != (instance != nil) { - t.Errorf("Expected instance exists: %t, got instance: %v", tc.exists, instance) + if tc.exists != (len(instances) > 0) { + t.Errorf("Expected instance exists: %t, got instances: %v", tc.exists, instances) } }) } diff --git a/pkg/actuators/machine/stubs.go b/pkg/actuators/machine/stubs.go index d670cb070d..9559e6db64 100644 --- a/pkg/actuators/machine/stubs.go +++ b/pkg/actuators/machine/stubs.go @@ -167,7 +167,7 @@ func stubInstance(imageID, instanceID string) *ec2.Instance { ImageId: aws.String(imageID), InstanceId: aws.String(instanceID), State: &ec2.InstanceState{ - Name: aws.String("Running"), + Name: aws.String(ec2.InstanceStateNameRunning), Code: aws.Int64(16), }, LaunchTime: aws.Time(time.Now()), @@ -248,7 +248,7 @@ func stubReservation(imageID, instanceID string) *ec2.Reservation { ImageId: aws.String(imageID), InstanceId: aws.String(instanceID), State: &ec2.InstanceState{ - Name: aws.String("Running"), + Name: aws.String(ec2.InstanceStateNameRunning), Code: aws.Int64(16), }, LaunchTime: aws.Time(time.Now()), diff --git a/pkg/actuators/machine/utils.go b/pkg/actuators/machine/utils.go index 4d7090d8b2..865d83933d 100644 --- a/pkg/actuators/machine/utils.go +++ b/pkg/actuators/machine/utils.go @@ -65,24 +65,38 @@ func getStoppedInstances(machine *machinev1.Machine, client awsclient.Client) ([ // getExistingInstances returns all instances not terminated func getExistingInstances(machine *machinev1.Machine, client awsclient.Client) ([]*ec2.Instance, error) { - instances, err := getInstances(machine, client, nil) - if err != nil { - return nil, err + return getInstances(machine, client, existingInstanceStates()) +} + +func getExistingInstanceByID(id string, client awsclient.Client) (*ec2.Instance, error) { + return getInstanceByID(id, client, existingInstanceStates()) +} + +func instanceHasAllowedState(instance *ec2.Instance, instanceStateFilter []*string) error { + if instance.InstanceId == nil { + return fmt.Errorf("instance has nil ID") } - var existingInstances []*ec2.Instance - for key := range instances { - if (instances[key].State) != nil { - if aws.StringValue(instances[key].State.Name) != ec2.InstanceStateNameTerminated { - existingInstances = append(existingInstances, instances[key]) - } + if instance.State == nil { + return fmt.Errorf("instance %s has nil state", *instance.InstanceId) + } + + if len(instanceStateFilter) == 0 { + return nil + } + + actualState := aws.StringValue(instance.State.Name) + for _, allowedState := range instanceStateFilter { + if aws.StringValue(allowedState) == actualState { + return nil } } - return existingInstances, nil -} -func getExistingInstanceByID(id string, client awsclient.Client) (*ec2.Instance, error) { - return getInstanceByID(id, client, existingInstanceStates()) + allowedStates := make([]string, 0, len(instanceStateFilter)) + for _, allowedState := range instanceStateFilter { + allowedStates = append(allowedStates, aws.StringValue(allowedState)) + } + return fmt.Errorf("instance %s state %q is not in %s", *instance.InstanceId, actualState, strings.Join(allowedStates, ", ")) } // getInstanceByID returns the instance with the given ID if it exists. @@ -112,26 +126,7 @@ func getInstanceByID(id string, client awsclient.Client, instanceStateFilter []* instance := reservation.Instances[0] - if len(instanceStateFilter) == 0 { - return instance, nil - } - - if instance.State == nil { - return nil, fmt.Errorf("instance %s has nil state", id) - } - - actualState := aws.StringValue(instance.State.Name) - for _, allowedState := range instanceStateFilter { - if aws.StringValue(allowedState) == actualState { - return instance, nil - } - } - - allowedStates := make([]string, 0, len(instanceStateFilter)) - for _, allowedState := range instanceStateFilter { - allowedStates = append(allowedStates, aws.StringValue(allowedState)) - } - return instance, fmt.Errorf("instance %s state %q is not in %s", id, actualState, strings.Join(allowedStates, ", ")) + return instance, instanceHasAllowedState(instance, instanceStateFilter) } // getInstances returns all instances that have a tag matching our machine name, @@ -149,12 +144,6 @@ func getInstances(machine *machinev1.Machine, client awsclient.Client, instanceS }, clusterFilter(clusterID), } - if instanceStateFilter != nil { - requestFilters = append(requestFilters, &ec2.Filter{ - Name: aws.String("instance-state-name"), - Values: instanceStateFilter, - }) - } request := &ec2.DescribeInstancesInput{ Filters: requestFilters, @@ -166,9 +155,15 @@ func getInstances(machine *machinev1.Machine, client awsclient.Client, instanceS } instances := make([]*ec2.Instance, 0, len(result.Reservations)) + for _, reservation := range result.Reservations { for _, instance := range reservation.Instances { - instances = append(instances, instance) + err := instanceHasAllowedState(instance, instanceStateFilter) + if err != nil { + glog.Errorf("Excluding instance matching %s: %v", machine.Name, err) + } else { + instances = append(instances, instance) + } } } diff --git a/pkg/client/fake/fake.go b/pkg/client/fake/fake.go index bb9c6ce5ab..d4377fd9c6 100644 --- a/pkg/client/fake/fake.go +++ b/pkg/client/fake/fake.go @@ -81,7 +81,7 @@ func (c *awsClient) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.D ImageId: aws.String("ami-a9acbbd6"), InstanceId: aws.String("i-02fcb933c5da7085c"), State: &ec2.InstanceState{ - Name: aws.String("Running"), + Name: aws.String(ec2.InstanceStateNameRunning), Code: aws.Int64(16), }, LaunchTime: aws.Time(time.Now()), From 1d28bb6234a09ca0e072827d82bc568e22fc35ee Mon Sep 17 00:00:00 2001 From: Enxebre Date: Mon, 18 Nov 2019 15:10:54 +0100 Subject: [PATCH 09/14] Use getMachineInstances to find instances during update() call --- pkg/actuators/machine/actuator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index 6fe17af748..e1e6f8da58 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -424,7 +424,7 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m return a.handleMachineError(machine, err, updateEventAction) } // Get all instances not terminated. - existingInstances, err := getExistingInstances(machine, client) + existingInstances, err := a.getMachineInstances(cluster, machine) if err != nil { glog.Errorf("%s: error getting existing instances: %v", machine.Name, err) return err From 9f07ffc4ab35053ecce10ba9c33d5137038688a7 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 19 Nov 2019 21:28:49 -0800 Subject: [PATCH 10/14] pkg/actuators/machine/actuator: Drop setMachineCloudProviderSpecifics comment The updateProviderID comment was probably a copy/paste error when setMachineCloudProviderSpecifics landed in 716083f839 (Set additional machine annotations/labels to get pretty machine output, 2019-07-23, openshift/cluster-api-provider-aws#242). I haven't read through the function carefully enough to be able to propose a comment that adds much beyond the function name, which is fairly clear on its own. But I know the updateProviderID comment is not a good fit ;). --- pkg/actuators/machine/actuator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index e1e6f8da58..5449ee664f 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -130,7 +130,6 @@ func (a *Actuator) Create(context context.Context, cluster *clusterv1.Cluster, m return a.updateStatus(updatedMachine, instance) } -// updateProviderID adds providerID in the machine spec func (a *Actuator) setMachineCloudProviderSpecifics(machine *machinev1.Machine, instance *ec2.Instance) (*machinev1.Machine, error) { if instance == nil { return machine, nil From 73bb0152443b793aa74aa047eae8e41a41cb9397 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 18 Nov 2019 15:17:20 -0800 Subject: [PATCH 11/14] pkg/actuators/machine/actuator: Eventual-consistency wait for machines Avoid [1]: $ curl -s https://storage.googleapis.com/origin-ci-test/logs/release-openshift-openshift-ansible-e2e-aws-scaleup-rhel7-4.3/255/artifacts/e2e-aws-scaleup-rhel7/must-gather/registry-svc-ci-openshift-org-ci-op-b24mfpd6-stable-sha256-64c63eedf863406fbc6c7515026f909a7221472cf70283708fb7010dd5e6139e/namespaces/openshift-machine-api/pods/machine-api-controllers-785b4687bf-wspnd/machine-controller/machine-controller/logs/current.log | grep -B5 'ci-op-b24mfpd6-6df53-m62pz-worker-us-east-1a-centos-sl696: found 0 existing instances for machine' 2019-11-18T01:08:14.686879882Z I1118 01:08:14.686811 1 actuator.go:552] ci-op-b24mfpd6-6df53-m62pz-worker-us-east-1a-centos-sl696: Found instance by id: i-04e60273f38857eed 2019-11-18T01:08:14.686879882Z I1118 01:08:14.686839 1 actuator.go:502] ci-op-b24mfpd6-6df53-m62pz-worker-us-east-1a-centos-sl696: Instance exists as "i-04e60273f38857eed" 2019-11-18T01:08:14.686879882Z I1118 01:08:14.686853 1 controller.go:284] Reconciling machine "ci-op-b24mfpd6-6df53-m62pz-worker-us-east-1a-centos-sl696" triggers idempotent update 2019-11-18T01:08:14.686879882Z I1118 01:08:14.686862 1 actuator.go:406] ci-op-b24mfpd6-6df53-m62pz-worker-us-east-1a-centos-sl696: updating machine 2019-11-18T01:08:14.68696548Z I1118 01:08:14.686938 1 actuator.go:414] ci-op-b24mfpd6-6df53-m62pz-worker-us-east-1a-centos-sl696: obtaining EC2 client for region 2019-11-18T01:08:14.741237517Z I1118 01:08:14.741189 1 actuator.go:430] ci-op-b24mfpd6-6df53-m62pz-worker-us-east-1a-centos-sl696: found 0 existing instances for machine by waiting a bit for AWS to resolve any eventual-consistency issues [2] before giving up on a machine and claiming it failed. With this commit, we'll try again until requeueAfterSeconds have passed since the last status update when we have a machine ID to look for (from creation). The efficiency of the retry request is weakened a bit by the AWS provider / machine controller pairing apparently ignoring requeue delays [3]. And if we respected retries like that, we probably wouldn't hit the guard this commit is adding. But a busy loop here is still better than abandoning the machine, and we don't have a better fix for the ignored requeue delay in the pipe. Personally, I'd rather tie this guard to the Provisioning and Provisioned machine phases, or the MachineCreationSucceeded condition, or some such to avoid excessive delays if folks were bumping the machine status and LastUpdated for other reasons. But Alberto points out that machine phases are supposed to be a higher-level abstraction that doesn't leak down into the provider [4] and he prefers switching on ProviderID [5], so that's what we have in this commit. [1]: https://bugzilla.redhat.com/show_bug.cgi?id=1772163#c11 [2]: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/query-api-troubleshooting.html#eventual-consistency [3]: https://bugzilla.redhat.com/show_bug.cgi?id=1772192 [4]: https://github.com/openshift/cluster-api-provider-aws/pull/271#discussion_r347828973 [5]: https://github.com/openshift/cluster-api-provider-aws/pull/271#discussion_r348071993 --- pkg/actuators/machine/actuator.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index 5449ee664f..224bcb8103 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -434,6 +434,11 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m // Parent controller should prevent this from ever happening by calling Exists and then Create, // but instance could be deleted between the two calls. if existingLen == 0 { + if machine.Spec.ProviderID != nil && (machine.Status.LastUpdated == nil || machine.Status.LastUpdated.Add(requeueAfterSeconds*time.Second).After(time.Now())) { + glog.Infof("%s: Possible eventual-consistency discrepancy; returning an error to requeue", machine.Name) + return &clustererror.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second} + } + glog.Warningf("%s: attempted to update machine but no instances found", machine.Name) a.handleMachineError(machine, mapierrors.UpdateMachine("no instance found, reason unknown"), updateEventAction) From 6490dfcfe817ac273173fa8237c082c10c4a1c99 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 22 Nov 2019 15:52:25 -0800 Subject: [PATCH 12/14] pkg/actuators/machine: Eventual-consistency guards for Exists/Describe Same guard Update grew in openshift/cluster-api-provider-aws@09c1fe9cee9e68 (pkg/actuators/machine/actuator: Eventual-consistency wait for machines, 2019-11-18, openshift/cluster-api-provider-aws#271), but now for Exists and Describe. This should protect us from flows like [1]: $ curl -s https://storage.googleapis.com/origin-ci-test/pr-logs/pull/openshift_installer/2712/pull-ci-openshift-installer-master-e2e-aws-scaleup-rhel7/3059/artifacts/e2e-aws-scaleup-rhel7/must-gather/registry-svc-ci-openshift-org-ci-op-nr00dj4c-stable-sha256-b6dffb34bfe5c198d7e9ad032322277c2a129e45a4e7f2084a83ab560530f683/namespaces/openshift-machine-api/pods/machine-api-controllers-9bd7457fc-tfdmg/machine-controller/machine-controller/logs/current.log | grep ci-op-nr00dj4c-9de00-ls2x8-worker-us-east-1a-rhel-fdknz ... 2019-11-22T17:39:34.642709291Z I1122 17:39:34.642651 1 actuator.go:194] ci-op-nr00dj4c-9de00-ls2x8-worker-us-east-1a-rhel-fdknz: ProviderID updated at machine spec: aws:///us-east-1a/i-08b09536e36303450 ... 2019-11-22T17:39:34.660245157Z I1122 17:39:34.660202 1 actuator.go:668] ci-op-nr00dj4c-9de00-ls2x8-worker-us-east-1a-rhel-fdknz: Instance state still pending, returning an error to requeue 2019-11-22T17:39:34.660327796Z W1122 17:39:34.660285 1 controller.go:321] Failed to create machine "ci-op-nr00dj4c-9de00-ls2x8-worker-us-east-1a-rhel-fdknz": requeue in: 20s 2019-11-22T17:39:34.660426954Z I1122 17:39:34.660380 1 controller.go:164] Reconciling Machine "ci-op-nr00dj4c-9de00-ls2x8-worker-us-east-1a-rhel-fdknz" ... 2019-11-22T17:39:34.660426954Z I1122 17:39:34.660416 1 actuator.go:493] ci-op-nr00dj4c-9de00-ls2x8-worker-us-east-1a-rhel-fdknz: Checking if machine exists 2019-11-22T17:39:34.732221813Z I1122 17:39:34.732177 1 actuator.go:501] ci-op-nr00dj4c-9de00-ls2x8-worker-us-east-1a-rhel-fdknz: Instance does not exist 2019-11-22T17:39:34.732221813Z I1122 17:39:34.732202 1 controller.go:428] Machine "ci-op-nr00dj4c-9de00-ls2x8-worker-us-east-1a-rhel-fdknz" going into phase "Failed" ... Also, because the code was basically the same between Exists and Describe, I've refactored Exists to be a thin wrapper around Describe. It's unfortunate that Describe does not take a Context argument, but there's no regression because Exists was ignoring its Context argument before anyway ;). [1]: https://bugzilla.redhat.com/show_bug.cgi?id=1761882#c15 --- pkg/actuators/machine/actuator.go | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index 224bcb8103..08244792d1 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -493,21 +493,8 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m // Exists determines if the given machine currently exists. // A machine which is not terminated is considered as existing. func (a *Actuator) Exists(context context.Context, cluster *clusterv1.Cluster, machine *machinev1.Machine) (bool, error) { - glog.Infof("%s: Checking if machine exists", machine.Name) - - instances, err := a.getMachineInstances(cluster, machine) - if err != nil { - glog.Errorf("%s: Error getting existing instances: %v", machine.Name, err) - return false, err - } - if len(instances) == 0 { - glog.Infof("%s: Instance does not exist", machine.Name) - return false, nil - } - - // If more than one result was returned, it will be handled in Update. - glog.Infof("%s: Instance exists as %q", machine.Name, *instances[0].InstanceId) - return true, nil + instance, err := a.Describe(cluster, machine) + return instance != nil, err } // Describe provides information about machine's instance(s) @@ -516,10 +503,15 @@ func (a *Actuator) Describe(cluster *clusterv1.Cluster, machine *machinev1.Machi instances, err := a.getMachineInstances(cluster, machine) if err != nil { - glog.Errorf("%s: Error getting running instances: %v", machine.Name, err) + glog.Errorf("%s: Error getting existing instances: %v", machine.Name, err) return nil, err } if len(instances) == 0 { + if machine.Spec.ProviderID != nil && (machine.Status.LastUpdated == nil || machine.Status.LastUpdated.Add(requeueAfterSeconds*time.Second).After(time.Now())) { + glog.Infof("%s: Possible eventual-consistency discrepancy; returning an error to requeue", machine.Name) + return nil, &clustererror.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second} + } + glog.Infof("%s: Instance does not exist", machine.Name) return nil, nil } From 1a3ca55955ffcec2ece9ec7fb23fc64475981495 Mon Sep 17 00:00:00 2001 From: Michael Gugino Date: Mon, 9 Dec 2019 17:45:25 -0500 Subject: [PATCH 13/14] Ensure Spec.ProviderID is not empty string If the Spec.ProviderID field is present but an empty string, we may encounter issues. This is particularly the case in DR scenarios when a user is attempting to copy an existing machine in attempt to create one. --- pkg/actuators/machine/actuator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index 08244792d1..833ce1e0b4 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -434,7 +434,7 @@ func (a *Actuator) Update(context context.Context, cluster *clusterv1.Cluster, m // Parent controller should prevent this from ever happening by calling Exists and then Create, // but instance could be deleted between the two calls. if existingLen == 0 { - if machine.Spec.ProviderID != nil && (machine.Status.LastUpdated == nil || machine.Status.LastUpdated.Add(requeueAfterSeconds*time.Second).After(time.Now())) { + if machine.Spec.ProviderID != nil && *machine.Spec.ProviderID != "" && (machine.Status.LastUpdated == nil || machine.Status.LastUpdated.Add(requeueAfterSeconds*time.Second).After(time.Now())) { glog.Infof("%s: Possible eventual-consistency discrepancy; returning an error to requeue", machine.Name) return &clustererror.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second} } @@ -507,7 +507,7 @@ func (a *Actuator) Describe(cluster *clusterv1.Cluster, machine *machinev1.Machi return nil, err } if len(instances) == 0 { - if machine.Spec.ProviderID != nil && (machine.Status.LastUpdated == nil || machine.Status.LastUpdated.Add(requeueAfterSeconds*time.Second).After(time.Now())) { + if machine.Spec.ProviderID != nil && *machine.Spec.ProviderID != "" && (machine.Status.LastUpdated == nil || machine.Status.LastUpdated.Add(requeueAfterSeconds*time.Second).After(time.Now())) { glog.Infof("%s: Possible eventual-consistency discrepancy; returning an error to requeue", machine.Name) return nil, &clustererror.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second} } From 4f7fbd0c095012796c1243c88beea782e2368aaa Mon Sep 17 00:00:00 2001 From: Enxebre Date: Fri, 7 Feb 2020 13:01:46 +0100 Subject: [PATCH 14/14] Revendor to bring k8s.io/utils/pointer --- go.mod | 1 + vendor/golang.org/x/net/html/atom/gen.go | 712 ++++++++ vendor/golang.org/x/sys/unix/mkasm_darwin.go | 61 + vendor/golang.org/x/sys/unix/mkpost.go | 122 ++ vendor/golang.org/x/sys/unix/mksyscall.go | 407 +++++ .../x/sys/unix/mksyscall_aix_ppc.go | 415 +++++ .../x/sys/unix/mksyscall_aix_ppc64.go | 614 +++++++ .../x/sys/unix/mksyscall_solaris.go | 335 ++++ .../golang.org/x/sys/unix/mksysctl_openbsd.go | 355 ++++ vendor/golang.org/x/sys/unix/mksysnum.go | 190 +++ vendor/golang.org/x/sys/unix/types_aix.go | 237 +++ vendor/golang.org/x/sys/unix/types_darwin.go | 283 +++ .../golang.org/x/sys/unix/types_dragonfly.go | 263 +++ vendor/golang.org/x/sys/unix/types_freebsd.go | 356 ++++ vendor/golang.org/x/sys/unix/types_netbsd.go | 289 ++++ vendor/golang.org/x/sys/unix/types_openbsd.go | 282 +++ vendor/golang.org/x/sys/unix/types_solaris.go | 266 +++ .../x/text/encoding/charmap/maketables.go | 556 ++++++ .../x/text/encoding/htmlindex/gen.go | 173 ++ .../text/encoding/internal/identifier/gen.go | 142 ++ .../x/text/encoding/japanese/maketables.go | 161 ++ .../x/text/encoding/korean/maketables.go | 143 ++ .../encoding/simplifiedchinese/maketables.go | 161 ++ .../encoding/traditionalchinese/maketables.go | 140 ++ .../x/text/internal/language/compact/gen.go | 64 + .../internal/language/compact/gen_index.go | 113 ++ .../internal/language/compact/gen_parents.go | 54 + .../x/text/internal/language/gen.go | 1520 +++++++++++++++++ .../x/text/internal/language/gen_common.go | 20 + vendor/golang.org/x/text/language/gen.go | 305 ++++ vendor/golang.org/x/text/unicode/bidi/gen.go | 133 ++ .../x/text/unicode/bidi/gen_ranges.go | 57 + .../x/text/unicode/bidi/gen_trieval.go | 64 + .../x/text/unicode/norm/maketables.go | 986 +++++++++++ .../golang.org/x/text/unicode/norm/triegen.go | 117 ++ vendor/golang.org/x/text/width/gen.go | 115 ++ vendor/golang.org/x/text/width/gen_common.go | 96 ++ vendor/golang.org/x/text/width/gen_trieval.go | 34 + .../x/tools/go/gcexportdata/main.go | 99 ++ vendor/k8s.io/utils/pointer/OWNERS | 10 + vendor/k8s.io/utils/pointer/pointer.go | 86 + vendor/modules.txt | 337 ++-- 42 files changed, 10706 insertions(+), 168 deletions(-) create mode 100644 vendor/golang.org/x/net/html/atom/gen.go create mode 100644 vendor/golang.org/x/sys/unix/mkasm_darwin.go create mode 100644 vendor/golang.org/x/sys/unix/mkpost.go create mode 100644 vendor/golang.org/x/sys/unix/mksyscall.go create mode 100644 vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go create mode 100644 vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/mksyscall_solaris.go create mode 100644 vendor/golang.org/x/sys/unix/mksysctl_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/mksysnum.go create mode 100644 vendor/golang.org/x/sys/unix/types_aix.go create mode 100644 vendor/golang.org/x/sys/unix/types_darwin.go create mode 100644 vendor/golang.org/x/sys/unix/types_dragonfly.go create mode 100644 vendor/golang.org/x/sys/unix/types_freebsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_netbsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_solaris.go create mode 100644 vendor/golang.org/x/text/encoding/charmap/maketables.go create mode 100644 vendor/golang.org/x/text/encoding/htmlindex/gen.go create mode 100644 vendor/golang.org/x/text/encoding/internal/identifier/gen.go create mode 100644 vendor/golang.org/x/text/encoding/japanese/maketables.go create mode 100644 vendor/golang.org/x/text/encoding/korean/maketables.go create mode 100644 vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go create mode 100644 vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go create mode 100644 vendor/golang.org/x/text/internal/language/compact/gen.go create mode 100644 vendor/golang.org/x/text/internal/language/compact/gen_index.go create mode 100644 vendor/golang.org/x/text/internal/language/compact/gen_parents.go create mode 100644 vendor/golang.org/x/text/internal/language/gen.go create mode 100644 vendor/golang.org/x/text/internal/language/gen_common.go create mode 100644 vendor/golang.org/x/text/language/gen.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/gen.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/gen_ranges.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/gen_trieval.go create mode 100644 vendor/golang.org/x/text/unicode/norm/maketables.go create mode 100644 vendor/golang.org/x/text/unicode/norm/triegen.go create mode 100644 vendor/golang.org/x/text/width/gen.go create mode 100644 vendor/golang.org/x/text/width/gen_common.go create mode 100644 vendor/golang.org/x/text/width/gen_trieval.go create mode 100644 vendor/golang.org/x/tools/go/gcexportdata/main.go create mode 100644 vendor/k8s.io/utils/pointer/OWNERS create mode 100644 vendor/k8s.io/utils/pointer/pointer.go diff --git a/go.mod b/go.mod index c583b91c2f..7e702e9fa9 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible k8s.io/klog v0.3.1 k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058 // indirect + k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 sigs.k8s.io/controller-runtime v0.0.0-20190520212815-96b67f231945 sigs.k8s.io/controller-tools v0.2.0 ) diff --git a/vendor/golang.org/x/net/html/atom/gen.go b/vendor/golang.org/x/net/html/atom/gen.go new file mode 100644 index 0000000000..5d052781bc --- /dev/null +++ b/vendor/golang.org/x/net/html/atom/gen.go @@ -0,0 +1,712 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +//go:generate go run gen.go +//go:generate go run gen.go -test + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "io/ioutil" + "math/rand" + "os" + "sort" + "strings" +) + +// identifier converts s to a Go exported identifier. +// It converts "div" to "Div" and "accept-charset" to "AcceptCharset". +func identifier(s string) string { + b := make([]byte, 0, len(s)) + cap := true + for _, c := range s { + if c == '-' { + cap = true + continue + } + if cap && 'a' <= c && c <= 'z' { + c -= 'a' - 'A' + } + cap = false + b = append(b, byte(c)) + } + return string(b) +} + +var test = flag.Bool("test", false, "generate table_test.go") + +func genFile(name string, buf *bytes.Buffer) { + b, err := format.Source(buf.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := ioutil.WriteFile(name, b, 0644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func main() { + flag.Parse() + + var all []string + all = append(all, elements...) + all = append(all, attributes...) + all = append(all, eventHandlers...) + all = append(all, extra...) + sort.Strings(all) + + // uniq - lists have dups + w := 0 + for _, s := range all { + if w == 0 || all[w-1] != s { + all[w] = s + w++ + } + } + all = all[:w] + + if *test { + var buf bytes.Buffer + fmt.Fprintln(&buf, "// Code generated by go generate gen.go; DO NOT EDIT.\n") + fmt.Fprintln(&buf, "//go:generate go run gen.go -test\n") + fmt.Fprintln(&buf, "package atom\n") + fmt.Fprintln(&buf, "var testAtomList = []string{") + for _, s := range all { + fmt.Fprintf(&buf, "\t%q,\n", s) + } + fmt.Fprintln(&buf, "}") + + genFile("table_test.go", &buf) + return + } + + // Find hash that minimizes table size. + var best *table + for i := 0; i < 1000000; i++ { + if best != nil && 1<<(best.k-1) < len(all) { + break + } + h := rand.Uint32() + for k := uint(0); k <= 16; k++ { + if best != nil && k >= best.k { + break + } + var t table + if t.init(h, k, all) { + best = &t + break + } + } + } + if best == nil { + fmt.Fprintf(os.Stderr, "failed to construct string table\n") + os.Exit(1) + } + + // Lay out strings, using overlaps when possible. + layout := append([]string{}, all...) + + // Remove strings that are substrings of other strings + for changed := true; changed; { + changed = false + for i, s := range layout { + if s == "" { + continue + } + for j, t := range layout { + if i != j && t != "" && strings.Contains(s, t) { + changed = true + layout[j] = "" + } + } + } + } + + // Join strings where one suffix matches another prefix. + for { + // Find best i, j, k such that layout[i][len-k:] == layout[j][:k], + // maximizing overlap length k. + besti := -1 + bestj := -1 + bestk := 0 + for i, s := range layout { + if s == "" { + continue + } + for j, t := range layout { + if i == j { + continue + } + for k := bestk + 1; k <= len(s) && k <= len(t); k++ { + if s[len(s)-k:] == t[:k] { + besti = i + bestj = j + bestk = k + } + } + } + } + if bestk > 0 { + layout[besti] += layout[bestj][bestk:] + layout[bestj] = "" + continue + } + break + } + + text := strings.Join(layout, "") + + atom := map[string]uint32{} + for _, s := range all { + off := strings.Index(text, s) + if off < 0 { + panic("lost string " + s) + } + atom[s] = uint32(off<<8 | len(s)) + } + + var buf bytes.Buffer + // Generate the Go code. + fmt.Fprintln(&buf, "// Code generated by go generate gen.go; DO NOT EDIT.\n") + fmt.Fprintln(&buf, "//go:generate go run gen.go\n") + fmt.Fprintln(&buf, "package atom\n\nconst (") + + // compute max len + maxLen := 0 + for _, s := range all { + if maxLen < len(s) { + maxLen = len(s) + } + fmt.Fprintf(&buf, "\t%s Atom = %#x\n", identifier(s), atom[s]) + } + fmt.Fprintln(&buf, ")\n") + + fmt.Fprintf(&buf, "const hash0 = %#x\n\n", best.h0) + fmt.Fprintf(&buf, "const maxAtomLen = %d\n\n", maxLen) + + fmt.Fprintf(&buf, "var table = [1<<%d]Atom{\n", best.k) + for i, s := range best.tab { + if s == "" { + continue + } + fmt.Fprintf(&buf, "\t%#x: %#x, // %s\n", i, atom[s], s) + } + fmt.Fprintf(&buf, "}\n") + datasize := (1 << best.k) * 4 + + fmt.Fprintln(&buf, "const atomText =") + textsize := len(text) + for len(text) > 60 { + fmt.Fprintf(&buf, "\t%q +\n", text[:60]) + text = text[60:] + } + fmt.Fprintf(&buf, "\t%q\n\n", text) + + genFile("table.go", &buf) + + fmt.Fprintf(os.Stdout, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize) +} + +type byLen []string + +func (x byLen) Less(i, j int) bool { return len(x[i]) > len(x[j]) } +func (x byLen) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x byLen) Len() int { return len(x) } + +// fnv computes the FNV hash with an arbitrary starting value h. +func fnv(h uint32, s string) uint32 { + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} + +// A table represents an attempt at constructing the lookup table. +// The lookup table uses cuckoo hashing, meaning that each string +// can be found in one of two positions. +type table struct { + h0 uint32 + k uint + mask uint32 + tab []string +} + +// hash returns the two hashes for s. +func (t *table) hash(s string) (h1, h2 uint32) { + h := fnv(t.h0, s) + h1 = h & t.mask + h2 = (h >> 16) & t.mask + return +} + +// init initializes the table with the given parameters. +// h0 is the initial hash value, +// k is the number of bits of hash value to use, and +// x is the list of strings to store in the table. +// init returns false if the table cannot be constructed. +func (t *table) init(h0 uint32, k uint, x []string) bool { + t.h0 = h0 + t.k = k + t.tab = make([]string, 1< len(t.tab) { + return false + } + s := t.tab[i] + h1, h2 := t.hash(s) + j := h1 + h2 - i + if t.tab[j] != "" && !t.push(j, depth+1) { + return false + } + t.tab[j] = s + return true +} + +// The lists of element names and attribute keys were taken from +// https://html.spec.whatwg.org/multipage/indices.html#index +// as of the "HTML Living Standard - Last Updated 16 April 2018" version. + +// "command", "keygen" and "menuitem" have been removed from the spec, +// but are kept here for backwards compatibility. +var elements = []string{ + "a", + "abbr", + "address", + "area", + "article", + "aside", + "audio", + "b", + "base", + "bdi", + "bdo", + "blockquote", + "body", + "br", + "button", + "canvas", + "caption", + "cite", + "code", + "col", + "colgroup", + "command", + "data", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "div", + "dl", + "dt", + "em", + "embed", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hgroup", + "hr", + "html", + "i", + "iframe", + "img", + "input", + "ins", + "kbd", + "keygen", + "label", + "legend", + "li", + "link", + "main", + "map", + "mark", + "menu", + "menuitem", + "meta", + "meter", + "nav", + "noscript", + "object", + "ol", + "optgroup", + "option", + "output", + "p", + "param", + "picture", + "pre", + "progress", + "q", + "rp", + "rt", + "ruby", + "s", + "samp", + "script", + "section", + "select", + "slot", + "small", + "source", + "span", + "strong", + "style", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "template", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "tr", + "track", + "u", + "ul", + "var", + "video", + "wbr", +} + +// https://html.spec.whatwg.org/multipage/indices.html#attributes-3 +// +// "challenge", "command", "contextmenu", "dropzone", "icon", "keytype", "mediagroup", +// "radiogroup", "spellcheck", "scoped", "seamless", "sortable" and "sorted" have been removed from the spec, +// but are kept here for backwards compatibility. +var attributes = []string{ + "abbr", + "accept", + "accept-charset", + "accesskey", + "action", + "allowfullscreen", + "allowpaymentrequest", + "allowusermedia", + "alt", + "as", + "async", + "autocomplete", + "autofocus", + "autoplay", + "challenge", + "charset", + "checked", + "cite", + "class", + "color", + "cols", + "colspan", + "command", + "content", + "contenteditable", + "contextmenu", + "controls", + "coords", + "crossorigin", + "data", + "datetime", + "default", + "defer", + "dir", + "dirname", + "disabled", + "download", + "draggable", + "dropzone", + "enctype", + "for", + "form", + "formaction", + "formenctype", + "formmethod", + "formnovalidate", + "formtarget", + "headers", + "height", + "hidden", + "high", + "href", + "hreflang", + "http-equiv", + "icon", + "id", + "inputmode", + "integrity", + "is", + "ismap", + "itemid", + "itemprop", + "itemref", + "itemscope", + "itemtype", + "keytype", + "kind", + "label", + "lang", + "list", + "loop", + "low", + "manifest", + "max", + "maxlength", + "media", + "mediagroup", + "method", + "min", + "minlength", + "multiple", + "muted", + "name", + "nomodule", + "nonce", + "novalidate", + "open", + "optimum", + "pattern", + "ping", + "placeholder", + "playsinline", + "poster", + "preload", + "radiogroup", + "readonly", + "referrerpolicy", + "rel", + "required", + "reversed", + "rows", + "rowspan", + "sandbox", + "spellcheck", + "scope", + "scoped", + "seamless", + "selected", + "shape", + "size", + "sizes", + "sortable", + "sorted", + "slot", + "span", + "spellcheck", + "src", + "srcdoc", + "srclang", + "srcset", + "start", + "step", + "style", + "tabindex", + "target", + "title", + "translate", + "type", + "typemustmatch", + "updateviacache", + "usemap", + "value", + "width", + "workertype", + "wrap", +} + +// "onautocomplete", "onautocompleteerror", "onmousewheel", +// "onshow" and "onsort" have been removed from the spec, +// but are kept here for backwards compatibility. +var eventHandlers = []string{ + "onabort", + "onautocomplete", + "onautocompleteerror", + "onauxclick", + "onafterprint", + "onbeforeprint", + "onbeforeunload", + "onblur", + "oncancel", + "oncanplay", + "oncanplaythrough", + "onchange", + "onclick", + "onclose", + "oncontextmenu", + "oncopy", + "oncuechange", + "oncut", + "ondblclick", + "ondrag", + "ondragend", + "ondragenter", + "ondragexit", + "ondragleave", + "ondragover", + "ondragstart", + "ondrop", + "ondurationchange", + "onemptied", + "onended", + "onerror", + "onfocus", + "onhashchange", + "oninput", + "oninvalid", + "onkeydown", + "onkeypress", + "onkeyup", + "onlanguagechange", + "onload", + "onloadeddata", + "onloadedmetadata", + "onloadend", + "onloadstart", + "onmessage", + "onmessageerror", + "onmousedown", + "onmouseenter", + "onmouseleave", + "onmousemove", + "onmouseout", + "onmouseover", + "onmouseup", + "onmousewheel", + "onwheel", + "onoffline", + "ononline", + "onpagehide", + "onpageshow", + "onpaste", + "onpause", + "onplay", + "onplaying", + "onpopstate", + "onprogress", + "onratechange", + "onreset", + "onresize", + "onrejectionhandled", + "onscroll", + "onsecuritypolicyviolation", + "onseeked", + "onseeking", + "onselect", + "onshow", + "onsort", + "onstalled", + "onstorage", + "onsubmit", + "onsuspend", + "ontimeupdate", + "ontoggle", + "onunhandledrejection", + "onunload", + "onvolumechange", + "onwaiting", +} + +// extra are ad-hoc values not covered by any of the lists above. +var extra = []string{ + "acronym", + "align", + "annotation", + "annotation-xml", + "applet", + "basefont", + "bgsound", + "big", + "blink", + "center", + "color", + "desc", + "face", + "font", + "foreignObject", // HTML is case-insensitive, but SVG-embedded-in-HTML is case-sensitive. + "foreignobject", + "frame", + "frameset", + "image", + "isindex", + "listing", + "malignmark", + "marquee", + "math", + "mglyph", + "mi", + "mn", + "mo", + "ms", + "mtext", + "nobr", + "noembed", + "noframes", + "plaintext", + "prompt", + "public", + "rb", + "rtc", + "spacer", + "strike", + "svg", + "system", + "tt", + "xmp", +} diff --git a/vendor/golang.org/x/sys/unix/mkasm_darwin.go b/vendor/golang.org/x/sys/unix/mkasm_darwin.go new file mode 100644 index 0000000000..4548b993db --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mkasm_darwin.go @@ -0,0 +1,61 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// mkasm_darwin.go generates assembly trampolines to call libSystem routines from Go. +//This program must be run after mksyscall.go. +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "strings" +) + +func main() { + in1, err := ioutil.ReadFile("syscall_darwin.go") + if err != nil { + log.Fatalf("can't open syscall_darwin.go: %s", err) + } + arch := os.Args[1] + in2, err := ioutil.ReadFile(fmt.Sprintf("syscall_darwin_%s.go", arch)) + if err != nil { + log.Fatalf("can't open syscall_darwin_%s.go: %s", arch, err) + } + in3, err := ioutil.ReadFile(fmt.Sprintf("zsyscall_darwin_%s.go", arch)) + if err != nil { + log.Fatalf("can't open zsyscall_darwin_%s.go: %s", arch, err) + } + in := string(in1) + string(in2) + string(in3) + + trampolines := map[string]bool{} + + var out bytes.Buffer + + fmt.Fprintf(&out, "// go run mkasm_darwin.go %s\n", strings.Join(os.Args[1:], " ")) + fmt.Fprintf(&out, "// Code generated by the command above; DO NOT EDIT.\n") + fmt.Fprintf(&out, "\n") + fmt.Fprintf(&out, "// +build go1.12\n") + fmt.Fprintf(&out, "\n") + fmt.Fprintf(&out, "#include \"textflag.h\"\n") + for _, line := range strings.Split(in, "\n") { + if !strings.HasPrefix(line, "func ") || !strings.HasSuffix(line, "_trampoline()") { + continue + } + fn := line[5 : len(line)-13] + if !trampolines[fn] { + trampolines[fn] = true + fmt.Fprintf(&out, "TEXT ·%s_trampoline(SB),NOSPLIT,$0-0\n", fn) + fmt.Fprintf(&out, "\tJMP\t%s(SB)\n", fn) + } + } + err = ioutil.WriteFile(fmt.Sprintf("zsyscall_darwin_%s.s", arch), out.Bytes(), 0644) + if err != nil { + log.Fatalf("can't write zsyscall_darwin_%s.s: %s", arch, err) + } +} diff --git a/vendor/golang.org/x/sys/unix/mkpost.go b/vendor/golang.org/x/sys/unix/mkpost.go new file mode 100644 index 0000000000..4d5b531b50 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mkpost.go @@ -0,0 +1,122 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// mkpost processes the output of cgo -godefs to +// modify the generated types. It is used to clean up +// the sys API in an architecture specific manner. +// +// mkpost is run after cgo -godefs; see README.md. +package main + +import ( + "bytes" + "fmt" + "go/format" + "io/ioutil" + "log" + "os" + "regexp" +) + +func main() { + // Get the OS and architecture (using GOARCH_TARGET if it exists) + goos := os.Getenv("GOOS") + goarch := os.Getenv("GOARCH_TARGET") + if goarch == "" { + goarch = os.Getenv("GOARCH") + } + // Check that we are using the Docker-based build system if we should be. + if goos == "linux" { + if os.Getenv("GOLANG_SYS_BUILD") != "docker" { + os.Stderr.WriteString("In the Docker-based build system, mkpost should not be called directly.\n") + os.Stderr.WriteString("See README.md\n") + os.Exit(1) + } + } + + b, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Fatal(err) + } + + if goos == "aix" { + // Replace type of Atim, Mtim and Ctim by Timespec in Stat_t + // to avoid having both StTimespec and Timespec. + sttimespec := regexp.MustCompile(`_Ctype_struct_st_timespec`) + b = sttimespec.ReplaceAll(b, []byte("Timespec")) + } + + // Intentionally export __val fields in Fsid and Sigset_t + valRegex := regexp.MustCompile(`type (Fsid|Sigset_t) struct {(\s+)X__val(\s+\S+\s+)}`) + b = valRegex.ReplaceAll(b, []byte("type $1 struct {${2}Val$3}")) + + // Intentionally export __fds_bits field in FdSet + fdSetRegex := regexp.MustCompile(`type (FdSet) struct {(\s+)X__fds_bits(\s+\S+\s+)}`) + b = fdSetRegex.ReplaceAll(b, []byte("type $1 struct {${2}Bits$3}")) + + // If we have empty Ptrace structs, we should delete them. Only s390x emits + // nonempty Ptrace structs. + ptraceRexexp := regexp.MustCompile(`type Ptrace((Psw|Fpregs|Per) struct {\s*})`) + b = ptraceRexexp.ReplaceAll(b, nil) + + // Replace the control_regs union with a blank identifier for now. + controlRegsRegex := regexp.MustCompile(`(Control_regs)\s+\[0\]uint64`) + b = controlRegsRegex.ReplaceAll(b, []byte("_ [0]uint64")) + + // Remove fields that are added by glibc + // Note that this is unstable as the identifers are private. + removeFieldsRegex := regexp.MustCompile(`X__glibc\S*`) + b = removeFieldsRegex.ReplaceAll(b, []byte("_")) + + // Convert [65]int8 to [65]byte in Utsname members to simplify + // conversion to string; see golang.org/issue/20753 + convertUtsnameRegex := regexp.MustCompile(`((Sys|Node|Domain)name|Release|Version|Machine)(\s+)\[(\d+)\]u?int8`) + b = convertUtsnameRegex.ReplaceAll(b, []byte("$1$3[$4]byte")) + + // Convert [1024]int8 to [1024]byte in Ptmget members + convertPtmget := regexp.MustCompile(`([SC]n)(\s+)\[(\d+)\]u?int8`) + b = convertPtmget.ReplaceAll(b, []byte("$1[$3]byte")) + + // Remove spare fields (e.g. in Statx_t) + spareFieldsRegex := regexp.MustCompile(`X__spare\S*`) + b = spareFieldsRegex.ReplaceAll(b, []byte("_")) + + // Remove cgo padding fields + removePaddingFieldsRegex := regexp.MustCompile(`Pad_cgo_\d+`) + b = removePaddingFieldsRegex.ReplaceAll(b, []byte("_")) + + // Remove padding, hidden, or unused fields + removeFieldsRegex = regexp.MustCompile(`\b(X_\S+|Padding)`) + b = removeFieldsRegex.ReplaceAll(b, []byte("_")) + + // Remove the first line of warning from cgo + b = b[bytes.IndexByte(b, '\n')+1:] + // Modify the command in the header to include: + // mkpost, our own warning, and a build tag. + replacement := fmt.Sprintf(`$1 | go run mkpost.go +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s,%s`, goarch, goos) + cgoCommandRegex := regexp.MustCompile(`(cgo -godefs .*)`) + b = cgoCommandRegex.ReplaceAll(b, []byte(replacement)) + + // Rename Stat_t time fields + if goos == "freebsd" && goarch == "386" { + // Hide Stat_t.[AMCB]tim_ext fields + renameStatTimeExtFieldsRegex := regexp.MustCompile(`[AMCB]tim_ext`) + b = renameStatTimeExtFieldsRegex.ReplaceAll(b, []byte("_")) + } + renameStatTimeFieldsRegex := regexp.MustCompile(`([AMCB])(?:irth)?time?(?:spec)?\s+(Timespec|StTimespec)`) + b = renameStatTimeFieldsRegex.ReplaceAll(b, []byte("${1}tim ${2}")) + + // gofmt + b, err = format.Source(b) + if err != nil { + log.Fatal(err) + } + + os.Stdout.Write(b) +} diff --git a/vendor/golang.org/x/sys/unix/mksyscall.go b/vendor/golang.org/x/sys/unix/mksyscall.go new file mode 100644 index 0000000000..e4af9424e9 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksyscall.go @@ -0,0 +1,407 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +This program reads a file containing function prototypes +(like syscall_darwin.go) and generates system call bodies. +The prototypes are marked by lines beginning with "//sys" +and read like func declarations if //sys is replaced by func, but: + * The parameter lists must give a name for each argument. + This includes return parameters. + * The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + * If the return parameter is an error number, it must be named errno. + +A line beginning with //sysnb is like //sys, except that the +goroutine will not be suspended during the execution of the system +call. This must only be used for system calls which can never +block, as otherwise the system call could cause all goroutines to +hang. +*/ +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "regexp" + "strings" +) + +var ( + b32 = flag.Bool("b32", false, "32bit big-endian") + l32 = flag.Bool("l32", false, "32bit little-endian") + plan9 = flag.Bool("plan9", false, "plan9") + openbsd = flag.Bool("openbsd", false, "openbsd") + netbsd = flag.Bool("netbsd", false, "netbsd") + dragonfly = flag.Bool("dragonfly", false, "dragonfly") + arm = flag.Bool("arm", false, "arm") // 64-bit value should use (even, odd)-pair + tags = flag.String("tags", "", "build tags") + filename = flag.String("output", "", "output file name (standard output if omitted)") +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksyscall.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return *tags +} + +// Param is function parameter +type Param struct { + Name string + Type string +} + +// usage prints the program usage +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run mksyscall.go [-b32 | -l32] [-tags x,y] [file ...]\n") + os.Exit(1) +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []string { + list = strings.TrimSpace(list) + if list == "" { + return []string{} + } + return regexp.MustCompile(`\s*,\s*`).Split(list, -1) +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexp.MustCompile(`^(\S*) (\S*)$`).FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + return Param{ps[1], ps[2]} +} + +func main() { + // Get the OS and architecture (using GOARCH_TARGET if it exists) + goos := os.Getenv("GOOS") + if goos == "" { + fmt.Fprintln(os.Stderr, "GOOS not defined in environment") + os.Exit(1) + } + goarch := os.Getenv("GOARCH_TARGET") + if goarch == "" { + goarch = os.Getenv("GOARCH") + } + + // Check that we are using the Docker-based build system if we should + if goos == "linux" { + if os.Getenv("GOLANG_SYS_BUILD") != "docker" { + fmt.Fprintf(os.Stderr, "In the Docker-based build system, mksyscall should not be called directly.\n") + fmt.Fprintf(os.Stderr, "See README.md\n") + os.Exit(1) + } + } + + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + endianness := "" + if *b32 { + endianness = "big-endian" + } else if *l32 { + endianness = "little-endian" + } + + libc := false + if goos == "darwin" && strings.Contains(buildTags(), ",go1.12") { + libc = true + } + trampolines := map[string]bool{} + + text := "" + for _, path := range flag.Args() { + file, err := os.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + t := s.Text() + t = strings.TrimSpace(t) + t = regexp.MustCompile(`\s+`).ReplaceAllString(t, ` `) + nonblock := regexp.MustCompile(`^\/\/sysnb `).FindStringSubmatch(t) + if regexp.MustCompile(`^\/\/sys `).FindStringSubmatch(t) == nil && nonblock == nil { + continue + } + + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, errno error) + // Split into name, in params, out params. + f := regexp.MustCompile(`^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*((?i)SYS_[A-Z0-9_]+))?$`).FindStringSubmatch(t) + if f == nil { + fmt.Fprintf(os.Stderr, "%s:%s\nmalformed //sys declaration\n", path, t) + os.Exit(1) + } + funct, inps, outps, sysname := f[2], f[3], f[4], f[5] + + // ClockGettime doesn't have a syscall number on Darwin, only generate libc wrappers. + if goos == "darwin" && !libc && funct == "ClockGettime" { + continue + } + + // Split argument lists on comma. + in := parseParamList(inps) + out := parseParamList(outps) + + // Try in vain to keep people from editing this file. + // The theory is that they jump into the middle of the file + // without reading the header. + text += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + + // Go function header. + outDecl := "" + if len(out) > 0 { + outDecl = fmt.Sprintf(" (%s)", strings.Join(out, ", ")) + } + text += fmt.Sprintf("func %s(%s)%s {\n", funct, strings.Join(in, ", "), outDecl) + + // Check if err return available + errvar := "" + for _, param := range out { + p := parseParam(param) + if p.Type == "error" { + errvar = p.Name + break + } + } + + // Prepare arguments to Syscall. + var args []string + n := 0 + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + args = append(args, "uintptr(unsafe.Pointer("+p.Name+"))") + } else if p.Type == "string" && errvar != "" { + text += fmt.Sprintf("\tvar _p%d *byte\n", n) + text += fmt.Sprintf("\t_p%d, %s = BytePtrFromString(%s)\n", n, errvar, p.Name) + text += fmt.Sprintf("\tif %s != nil {\n\t\treturn\n\t}\n", errvar) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + n++ + } else if p.Type == "string" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses string arguments, but has no error return\n") + text += fmt.Sprintf("\tvar _p%d *byte\n", n) + text += fmt.Sprintf("\t_p%d, _ = BytePtrFromString(%s)\n", n, p.Name) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + n++ + } else if regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type) != nil { + // Convert slice into pointer, length. + // Have to be careful not to take address of &a[0] if len == 0: + // pass dummy pointer in that case. + // Used to pass nil, but some OSes or simulators reject write(fd, nil, 0). + text += fmt.Sprintf("\tvar _p%d unsafe.Pointer\n", n) + text += fmt.Sprintf("\tif len(%s) > 0 {\n\t\t_p%d = unsafe.Pointer(&%s[0])\n\t}", p.Name, n, p.Name) + text += fmt.Sprintf(" else {\n\t\t_p%d = unsafe.Pointer(&_zero)\n\t}\n", n) + args = append(args, fmt.Sprintf("uintptr(_p%d)", n), fmt.Sprintf("uintptr(len(%s))", p.Name)) + n++ + } else if p.Type == "int64" && (*openbsd || *netbsd) { + args = append(args, "0") + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else if endianness == "little-endian" { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name)) + } + } else if p.Type == "int64" && *dragonfly { + if regexp.MustCompile(`^(?i)extp(read|write)`).FindStringSubmatch(funct) == nil { + args = append(args, "0") + } + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else if endianness == "little-endian" { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name)) + } + } else if (p.Type == "int64" || p.Type == "uint64") && endianness != "" { + if len(args)%2 == 1 && *arm { + // arm abi specifies 64-bit argument uses + // (even, odd) pair + args = append(args, "0") + } + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name)) + } + } + + // Determine which form to use; pad args with zeros. + asm := "Syscall" + if nonblock != nil { + if errvar == "" && goos == "linux" { + asm = "RawSyscallNoError" + } else { + asm = "RawSyscall" + } + } else { + if errvar == "" && goos == "linux" { + asm = "SyscallNoError" + } + } + if len(args) <= 3 { + for len(args) < 3 { + args = append(args, "0") + } + } else if len(args) <= 6 { + asm += "6" + for len(args) < 6 { + args = append(args, "0") + } + } else if len(args) <= 9 { + asm += "9" + for len(args) < 9 { + args = append(args, "0") + } + } else { + fmt.Fprintf(os.Stderr, "%s:%s too many arguments to system call\n", path, funct) + } + + // System call number. + if sysname == "" { + sysname = "SYS_" + funct + sysname = regexp.MustCompile(`([a-z])([A-Z])`).ReplaceAllString(sysname, `${1}_$2`) + sysname = strings.ToUpper(sysname) + } + + var libcFn string + if libc { + asm = "syscall_" + strings.ToLower(asm[:1]) + asm[1:] // internal syscall call + sysname = strings.TrimPrefix(sysname, "SYS_") // remove SYS_ + sysname = strings.ToLower(sysname) // lowercase + if sysname == "getdirentries64" { + // Special case - libSystem name and + // raw syscall name don't match. + sysname = "__getdirentries64" + } + libcFn = sysname + sysname = "funcPC(libc_" + sysname + "_trampoline)" + } + + // Actual call. + arglist := strings.Join(args, ", ") + call := fmt.Sprintf("%s(%s, %s)", asm, sysname, arglist) + + // Assign return values. + body := "" + ret := []string{"_", "_", "_"} + doErrno := false + for i := 0; i < len(out); i++ { + p := parseParam(out[i]) + reg := "" + if p.Name == "err" && !*plan9 { + reg = "e1" + ret[2] = reg + doErrno = true + } else if p.Name == "err" && *plan9 { + ret[0] = "r0" + ret[2] = "e1" + break + } else { + reg = fmt.Sprintf("r%d", i) + ret[i] = reg + } + if p.Type == "bool" { + reg = fmt.Sprintf("%s != 0", reg) + } + if p.Type == "int64" && endianness != "" { + // 64-bit number in r1:r0 or r0:r1. + if i+2 > len(out) { + fmt.Fprintf(os.Stderr, "%s:%s not enough registers for int64 return\n", path, funct) + } + if endianness == "big-endian" { + reg = fmt.Sprintf("int64(r%d)<<32 | int64(r%d)", i, i+1) + } else { + reg = fmt.Sprintf("int64(r%d)<<32 | int64(r%d)", i+1, i) + } + ret[i] = fmt.Sprintf("r%d", i) + ret[i+1] = fmt.Sprintf("r%d", i+1) + } + if reg != "e1" || *plan9 { + body += fmt.Sprintf("\t%s = %s(%s)\n", p.Name, p.Type, reg) + } + } + if ret[0] == "_" && ret[1] == "_" && ret[2] == "_" { + text += fmt.Sprintf("\t%s\n", call) + } else { + if errvar == "" && goos == "linux" { + // raw syscall without error on Linux, see golang.org/issue/22924 + text += fmt.Sprintf("\t%s, %s := %s\n", ret[0], ret[1], call) + } else { + text += fmt.Sprintf("\t%s, %s, %s := %s\n", ret[0], ret[1], ret[2], call) + } + } + text += body + + if *plan9 && ret[2] == "e1" { + text += "\tif int32(r0) == -1 {\n" + text += "\t\terr = e1\n" + text += "\t}\n" + } else if doErrno { + text += "\tif e1 != 0 {\n" + text += "\t\terr = errnoErr(e1)\n" + text += "\t}\n" + } + text += "\treturn\n" + text += "}\n\n" + + if libc && !trampolines[libcFn] { + // some system calls share a trampoline, like read and readlen. + trampolines[libcFn] = true + // Declare assembly trampoline. + text += fmt.Sprintf("func libc_%s_trampoline()\n", libcFn) + // Assembly trampoline calls the libc_* function, which this magic + // redirects to use the function from libSystem. + text += fmt.Sprintf("//go:linkname libc_%s libc_%s\n", libcFn, libcFn) + text += fmt.Sprintf("//go:cgo_import_dynamic libc_%s %s \"/usr/lib/libSystem.B.dylib\"\n", libcFn, libcFn) + text += "\n" + } + } + if err := s.Err(); err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + file.Close() + } + fmt.Printf(srcTemplate, cmdLine(), buildTags(), text) +} + +const srcTemplate = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package unix + +import ( + "syscall" + "unsafe" +) + +var _ syscall.Errno + +%s +` diff --git a/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go b/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go new file mode 100644 index 0000000000..3be3cdfc3b --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go @@ -0,0 +1,415 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +This program reads a file containing function prototypes +(like syscall_aix.go) and generates system call bodies. +The prototypes are marked by lines beginning with "//sys" +and read like func declarations if //sys is replaced by func, but: + * The parameter lists must give a name for each argument. + This includes return parameters. + * The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + * If the return parameter is an error number, it must be named err. + * If go func name needs to be different than its libc name, + * or the function is not in libc, name could be specified + * at the end, after "=" sign, like + //sys getsockopt(s int, level int, name int, val uintptr, vallen *_Socklen) (err error) = libsocket.getsockopt +*/ +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "regexp" + "strings" +) + +var ( + b32 = flag.Bool("b32", false, "32bit big-endian") + l32 = flag.Bool("l32", false, "32bit little-endian") + aix = flag.Bool("aix", false, "aix") + tags = flag.String("tags", "", "build tags") +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksyscall_aix_ppc.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return *tags +} + +// Param is function parameter +type Param struct { + Name string + Type string +} + +// usage prints the program usage +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run mksyscall_aix_ppc.go [-b32 | -l32] [-tags x,y] [file ...]\n") + os.Exit(1) +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []string { + list = strings.TrimSpace(list) + if list == "" { + return []string{} + } + return regexp.MustCompile(`\s*,\s*`).Split(list, -1) +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexp.MustCompile(`^(\S*) (\S*)$`).FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + return Param{ps[1], ps[2]} +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + endianness := "" + if *b32 { + endianness = "big-endian" + } else if *l32 { + endianness = "little-endian" + } + + pack := "" + text := "" + cExtern := "/*\n#include \n#include \n" + for _, path := range flag.Args() { + file, err := os.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + t := s.Text() + t = strings.TrimSpace(t) + t = regexp.MustCompile(`\s+`).ReplaceAllString(t, ` `) + if p := regexp.MustCompile(`^package (\S+)$`).FindStringSubmatch(t); p != nil && pack == "" { + pack = p[1] + } + nonblock := regexp.MustCompile(`^\/\/sysnb `).FindStringSubmatch(t) + if regexp.MustCompile(`^\/\/sys `).FindStringSubmatch(t) == nil && nonblock == nil { + continue + } + + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, err error) + // Split into name, in params, out params. + f := regexp.MustCompile(`^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*(?:(\w*)\.)?(\w*))?$`).FindStringSubmatch(t) + if f == nil { + fmt.Fprintf(os.Stderr, "%s:%s\nmalformed //sys declaration\n", path, t) + os.Exit(1) + } + funct, inps, outps, modname, sysname := f[2], f[3], f[4], f[5], f[6] + + // Split argument lists on comma. + in := parseParamList(inps) + out := parseParamList(outps) + + inps = strings.Join(in, ", ") + outps = strings.Join(out, ", ") + + // Try in vain to keep people from editing this file. + // The theory is that they jump into the middle of the file + // without reading the header. + text += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + + // Check if value return, err return available + errvar := "" + retvar := "" + rettype := "" + for _, param := range out { + p := parseParam(param) + if p.Type == "error" { + errvar = p.Name + } else { + retvar = p.Name + rettype = p.Type + } + } + + // System call name. + if sysname == "" { + sysname = funct + } + sysname = regexp.MustCompile(`([a-z])([A-Z])`).ReplaceAllString(sysname, `${1}_$2`) + sysname = strings.ToLower(sysname) // All libc functions are lowercase. + + cRettype := "" + if rettype == "unsafe.Pointer" { + cRettype = "uintptr_t" + } else if rettype == "uintptr" { + cRettype = "uintptr_t" + } else if regexp.MustCompile(`^_`).FindStringSubmatch(rettype) != nil { + cRettype = "uintptr_t" + } else if rettype == "int" { + cRettype = "int" + } else if rettype == "int32" { + cRettype = "int" + } else if rettype == "int64" { + cRettype = "long long" + } else if rettype == "uint32" { + cRettype = "unsigned int" + } else if rettype == "uint64" { + cRettype = "unsigned long long" + } else { + cRettype = "int" + } + if sysname == "exit" { + cRettype = "void" + } + + // Change p.Types to c + var cIn []string + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "string" { + cIn = append(cIn, "uintptr_t") + } else if regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t", "size_t") + } else if p.Type == "unsafe.Pointer" { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "uintptr" { + cIn = append(cIn, "uintptr_t") + } else if regexp.MustCompile(`^_`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "int" { + cIn = append(cIn, "int") + } else if p.Type == "int32" { + cIn = append(cIn, "int") + } else if p.Type == "int64" { + cIn = append(cIn, "long long") + } else if p.Type == "uint32" { + cIn = append(cIn, "unsigned int") + } else if p.Type == "uint64" { + cIn = append(cIn, "unsigned long long") + } else { + cIn = append(cIn, "int") + } + } + + if funct != "fcntl" && funct != "FcntlInt" && funct != "readlen" && funct != "writelen" { + if sysname == "select" { + // select is a keyword of Go. Its name is + // changed to c_select. + cExtern += "#define c_select select\n" + } + // Imports of system calls from libc + cExtern += fmt.Sprintf("%s %s", cRettype, sysname) + cIn := strings.Join(cIn, ", ") + cExtern += fmt.Sprintf("(%s);\n", cIn) + } + + // So file name. + if *aix { + if modname == "" { + modname = "libc.a/shr_64.o" + } else { + fmt.Fprintf(os.Stderr, "%s: only syscall using libc are available\n", funct) + os.Exit(1) + } + } + + strconvfunc := "C.CString" + + // Go function header. + if outps != "" { + outps = fmt.Sprintf(" (%s)", outps) + } + if text != "" { + text += "\n" + } + + text += fmt.Sprintf("func %s(%s)%s {\n", funct, strings.Join(in, ", "), outps) + + // Prepare arguments to Syscall. + var args []string + n := 0 + argN := 0 + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + args = append(args, "C.uintptr_t(uintptr(unsafe.Pointer("+p.Name+")))") + } else if p.Type == "string" && errvar != "" { + text += fmt.Sprintf("\t_p%d := uintptr(unsafe.Pointer(%s(%s)))\n", n, strconvfunc, p.Name) + args = append(args, fmt.Sprintf("C.uintptr_t(_p%d)", n)) + n++ + } else if p.Type == "string" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses string arguments, but has no error return\n") + text += fmt.Sprintf("\t_p%d := uintptr(unsafe.Pointer(%s(%s)))\n", n, strconvfunc, p.Name) + args = append(args, fmt.Sprintf("C.uintptr_t(_p%d)", n)) + n++ + } else if m := regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type); m != nil { + // Convert slice into pointer, length. + // Have to be careful not to take address of &a[0] if len == 0: + // pass nil in that case. + text += fmt.Sprintf("\tvar _p%d *%s\n", n, m[1]) + text += fmt.Sprintf("\tif len(%s) > 0 {\n\t\t_p%d = &%s[0]\n\t}\n", p.Name, n, p.Name) + args = append(args, fmt.Sprintf("C.uintptr_t(uintptr(unsafe.Pointer(_p%d)))", n)) + n++ + text += fmt.Sprintf("\tvar _p%d int\n", n) + text += fmt.Sprintf("\t_p%d = len(%s)\n", n, p.Name) + args = append(args, fmt.Sprintf("C.size_t(_p%d)", n)) + n++ + } else if p.Type == "int64" && endianness != "" { + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } + n++ + } else if p.Type == "bool" { + text += fmt.Sprintf("\tvar _p%d uint32\n", n) + text += fmt.Sprintf("\tif %s {\n\t\t_p%d = 1\n\t} else {\n\t\t_p%d = 0\n\t}\n", p.Name, n, n) + args = append(args, fmt.Sprintf("_p%d", n)) + } else if regexp.MustCompile(`^_`).FindStringSubmatch(p.Type) != nil { + args = append(args, fmt.Sprintf("C.uintptr_t(uintptr(%s))", p.Name)) + } else if p.Type == "unsafe.Pointer" { + args = append(args, fmt.Sprintf("C.uintptr_t(uintptr(%s))", p.Name)) + } else if p.Type == "int" { + if (argN == 2) && ((funct == "readlen") || (funct == "writelen")) { + args = append(args, fmt.Sprintf("C.size_t(%s)", p.Name)) + } else if argN == 0 && funct == "fcntl" { + args = append(args, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else if (argN == 2) && ((funct == "fcntl") || (funct == "FcntlInt")) { + args = append(args, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("C.int(%s)", p.Name)) + } + } else if p.Type == "int32" { + args = append(args, fmt.Sprintf("C.int(%s)", p.Name)) + } else if p.Type == "int64" { + args = append(args, fmt.Sprintf("C.longlong(%s)", p.Name)) + } else if p.Type == "uint32" { + args = append(args, fmt.Sprintf("C.uint(%s)", p.Name)) + } else if p.Type == "uint64" { + args = append(args, fmt.Sprintf("C.ulonglong(%s)", p.Name)) + } else if p.Type == "uintptr" { + args = append(args, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("C.int(%s)", p.Name)) + } + argN++ + } + + // Actual call. + arglist := strings.Join(args, ", ") + call := "" + if sysname == "exit" { + if errvar != "" { + call += "er :=" + } else { + call += "" + } + } else if errvar != "" { + call += "r0,er :=" + } else if retvar != "" { + call += "r0,_ :=" + } else { + call += "" + } + if sysname == "select" { + // select is a keyword of Go. Its name is + // changed to c_select. + call += fmt.Sprintf("C.c_%s(%s)", sysname, arglist) + } else { + call += fmt.Sprintf("C.%s(%s)", sysname, arglist) + } + + // Assign return values. + body := "" + for i := 0; i < len(out); i++ { + p := parseParam(out[i]) + reg := "" + if p.Name == "err" { + reg = "e1" + } else { + reg = "r0" + } + if reg != "e1" { + body += fmt.Sprintf("\t%s = %s(%s)\n", p.Name, p.Type, reg) + } + } + + // verify return + if sysname != "exit" && errvar != "" { + if regexp.MustCompile(`^uintptr`).FindStringSubmatch(cRettype) != nil { + body += "\tif (uintptr(r0) ==^uintptr(0) && er != nil) {\n" + body += fmt.Sprintf("\t\t%s = er\n", errvar) + body += "\t}\n" + } else { + body += "\tif (r0 ==-1 && er != nil) {\n" + body += fmt.Sprintf("\t\t%s = er\n", errvar) + body += "\t}\n" + } + } else if errvar != "" { + body += "\tif (er != nil) {\n" + body += fmt.Sprintf("\t\t%s = er\n", errvar) + body += "\t}\n" + } + + text += fmt.Sprintf("\t%s\n", call) + text += body + + text += "\treturn\n" + text += "}\n" + } + if err := s.Err(); err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + file.Close() + } + imp := "" + if pack != "unix" { + imp = "import \"golang.org/x/sys/unix\"\n" + + } + fmt.Printf(srcTemplate, cmdLine(), buildTags(), pack, cExtern, imp, text) +} + +const srcTemplate = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package %s + + +%s +*/ +import "C" +import ( + "unsafe" +) + + +%s + +%s +` diff --git a/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go b/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go new file mode 100644 index 0000000000..c960099517 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go @@ -0,0 +1,614 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +This program reads a file containing function prototypes +(like syscall_aix.go) and generates system call bodies. +The prototypes are marked by lines beginning with "//sys" +and read like func declarations if //sys is replaced by func, but: + * The parameter lists must give a name for each argument. + This includes return parameters. + * The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + * If the return parameter is an error number, it must be named err. + * If go func name needs to be different than its libc name, + * or the function is not in libc, name could be specified + * at the end, after "=" sign, like + //sys getsockopt(s int, level int, name int, val uintptr, vallen *_Socklen) (err error) = libsocket.getsockopt + + +This program will generate three files and handle both gc and gccgo implementation: + - zsyscall_aix_ppc64.go: the common part of each implementation (error handler, pointer creation) + - zsyscall_aix_ppc64_gc.go: gc part with //go_cgo_import_dynamic and a call to syscall6 + - zsyscall_aix_ppc64_gccgo.go: gccgo part with C function and conversion to C type. + + The generated code looks like this + +zsyscall_aix_ppc64.go +func asyscall(...) (n int, err error) { + // Pointer Creation + r1, e1 := callasyscall(...) + // Type Conversion + // Error Handler + return +} + +zsyscall_aix_ppc64_gc.go +//go:cgo_import_dynamic libc_asyscall asyscall "libc.a/shr_64.o" +//go:linkname libc_asyscall libc_asyscall +var asyscall syscallFunc + +func callasyscall(...) (r1 uintptr, e1 Errno) { + r1, _, e1 = syscall6(uintptr(unsafe.Pointer(&libc_asyscall)), "nb_args", ... ) + return +} + +zsyscall_aix_ppc64_ggcgo.go + +// int asyscall(...) + +import "C" + +func callasyscall(...) (r1 uintptr, e1 Errno) { + r1 = uintptr(C.asyscall(...)) + e1 = syscall.GetErrno() + return +} +*/ + +package main + +import ( + "bufio" + "flag" + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" +) + +var ( + b32 = flag.Bool("b32", false, "32bit big-endian") + l32 = flag.Bool("l32", false, "32bit little-endian") + aix = flag.Bool("aix", false, "aix") + tags = flag.String("tags", "", "build tags") +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksyscall_aix_ppc64.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return *tags +} + +// Param is function parameter +type Param struct { + Name string + Type string +} + +// usage prints the program usage +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run mksyscall_aix_ppc64.go [-b32 | -l32] [-tags x,y] [file ...]\n") + os.Exit(1) +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []string { + list = strings.TrimSpace(list) + if list == "" { + return []string{} + } + return regexp.MustCompile(`\s*,\s*`).Split(list, -1) +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexp.MustCompile(`^(\S*) (\S*)$`).FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + return Param{ps[1], ps[2]} +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + endianness := "" + if *b32 { + endianness = "big-endian" + } else if *l32 { + endianness = "little-endian" + } + + pack := "" + // GCCGO + textgccgo := "" + cExtern := "/*\n#include \n" + // GC + textgc := "" + dynimports := "" + linknames := "" + var vars []string + // COMMON + textcommon := "" + for _, path := range flag.Args() { + file, err := os.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + t := s.Text() + t = strings.TrimSpace(t) + t = regexp.MustCompile(`\s+`).ReplaceAllString(t, ` `) + if p := regexp.MustCompile(`^package (\S+)$`).FindStringSubmatch(t); p != nil && pack == "" { + pack = p[1] + } + nonblock := regexp.MustCompile(`^\/\/sysnb `).FindStringSubmatch(t) + if regexp.MustCompile(`^\/\/sys `).FindStringSubmatch(t) == nil && nonblock == nil { + continue + } + + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, err error) + // Split into name, in params, out params. + f := regexp.MustCompile(`^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*(?:(\w*)\.)?(\w*))?$`).FindStringSubmatch(t) + if f == nil { + fmt.Fprintf(os.Stderr, "%s:%s\nmalformed //sys declaration\n", path, t) + os.Exit(1) + } + funct, inps, outps, modname, sysname := f[2], f[3], f[4], f[5], f[6] + + // Split argument lists on comma. + in := parseParamList(inps) + out := parseParamList(outps) + + inps = strings.Join(in, ", ") + outps = strings.Join(out, ", ") + + if sysname == "" { + sysname = funct + } + + onlyCommon := false + if funct == "readlen" || funct == "writelen" || funct == "FcntlInt" || funct == "FcntlFlock" { + // This function call another syscall which is already implemented. + // Therefore, the gc and gccgo part must not be generated. + onlyCommon = true + } + + // Try in vain to keep people from editing this file. + // The theory is that they jump into the middle of the file + // without reading the header. + + textcommon += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + if !onlyCommon { + textgccgo += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + textgc += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + } + + // Check if value return, err return available + errvar := "" + rettype := "" + for _, param := range out { + p := parseParam(param) + if p.Type == "error" { + errvar = p.Name + } else { + rettype = p.Type + } + } + + sysname = regexp.MustCompile(`([a-z])([A-Z])`).ReplaceAllString(sysname, `${1}_$2`) + sysname = strings.ToLower(sysname) // All libc functions are lowercase. + + // GCCGO Prototype return type + cRettype := "" + if rettype == "unsafe.Pointer" { + cRettype = "uintptr_t" + } else if rettype == "uintptr" { + cRettype = "uintptr_t" + } else if regexp.MustCompile(`^_`).FindStringSubmatch(rettype) != nil { + cRettype = "uintptr_t" + } else if rettype == "int" { + cRettype = "int" + } else if rettype == "int32" { + cRettype = "int" + } else if rettype == "int64" { + cRettype = "long long" + } else if rettype == "uint32" { + cRettype = "unsigned int" + } else if rettype == "uint64" { + cRettype = "unsigned long long" + } else { + cRettype = "int" + } + if sysname == "exit" { + cRettype = "void" + } + + // GCCGO Prototype arguments type + var cIn []string + for i, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "string" { + cIn = append(cIn, "uintptr_t") + } else if regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t", "size_t") + } else if p.Type == "unsafe.Pointer" { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "uintptr" { + cIn = append(cIn, "uintptr_t") + } else if regexp.MustCompile(`^_`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "int" { + if (i == 0 || i == 2) && funct == "fcntl" { + // These fcntl arguments needs to be uintptr to be able to call FcntlInt and FcntlFlock + cIn = append(cIn, "uintptr_t") + } else { + cIn = append(cIn, "int") + } + + } else if p.Type == "int32" { + cIn = append(cIn, "int") + } else if p.Type == "int64" { + cIn = append(cIn, "long long") + } else if p.Type == "uint32" { + cIn = append(cIn, "unsigned int") + } else if p.Type == "uint64" { + cIn = append(cIn, "unsigned long long") + } else { + cIn = append(cIn, "int") + } + } + + if !onlyCommon { + // GCCGO Prototype Generation + // Imports of system calls from libc + if sysname == "select" { + // select is a keyword of Go. Its name is + // changed to c_select. + cExtern += "#define c_select select\n" + } + cExtern += fmt.Sprintf("%s %s", cRettype, sysname) + cIn := strings.Join(cIn, ", ") + cExtern += fmt.Sprintf("(%s);\n", cIn) + } + // GC Library name + if modname == "" { + modname = "libc.a/shr_64.o" + } else { + fmt.Fprintf(os.Stderr, "%s: only syscall using libc are available\n", funct) + os.Exit(1) + } + sysvarname := fmt.Sprintf("libc_%s", sysname) + + if !onlyCommon { + // GC Runtime import of function to allow cross-platform builds. + dynimports += fmt.Sprintf("//go:cgo_import_dynamic %s %s \"%s\"\n", sysvarname, sysname, modname) + // GC Link symbol to proc address variable. + linknames += fmt.Sprintf("//go:linkname %s %s\n", sysvarname, sysvarname) + // GC Library proc address variable. + vars = append(vars, sysvarname) + } + + strconvfunc := "BytePtrFromString" + strconvtype := "*byte" + + // Go function header. + if outps != "" { + outps = fmt.Sprintf(" (%s)", outps) + } + if textcommon != "" { + textcommon += "\n" + } + + textcommon += fmt.Sprintf("func %s(%s)%s {\n", funct, strings.Join(in, ", "), outps) + + // Prepare arguments tocall. + var argscommon []string // Arguments in the common part + var argscall []string // Arguments for call prototype + var argsgc []string // Arguments for gc call (with syscall6) + var argsgccgo []string // Arguments for gccgo call (with C.name_of_syscall) + n := 0 + argN := 0 + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + argscommon = append(argscommon, fmt.Sprintf("uintptr(unsafe.Pointer(%s))", p.Name)) + argscall = append(argscall, fmt.Sprintf("%s uintptr", p.Name)) + argsgc = append(argsgc, p.Name) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else if p.Type == "string" && errvar != "" { + textcommon += fmt.Sprintf("\tvar _p%d %s\n", n, strconvtype) + textcommon += fmt.Sprintf("\t_p%d, %s = %s(%s)\n", n, errvar, strconvfunc, p.Name) + textcommon += fmt.Sprintf("\tif %s != nil {\n\t\treturn\n\t}\n", errvar) + + argscommon = append(argscommon, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + argscall = append(argscall, fmt.Sprintf("_p%d uintptr ", n)) + argsgc = append(argsgc, fmt.Sprintf("_p%d", n)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(_p%d)", n)) + n++ + } else if p.Type == "string" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses string arguments, but has no error return\n") + textcommon += fmt.Sprintf("\tvar _p%d %s\n", n, strconvtype) + textcommon += fmt.Sprintf("\t_p%d, %s = %s(%s)\n", n, errvar, strconvfunc, p.Name) + textcommon += fmt.Sprintf("\tif %s != nil {\n\t\treturn\n\t}\n", errvar) + + argscommon = append(argscommon, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + argscall = append(argscall, fmt.Sprintf("_p%d uintptr", n)) + argsgc = append(argsgc, fmt.Sprintf("_p%d", n)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(_p%d)", n)) + n++ + } else if m := regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type); m != nil { + // Convert slice into pointer, length. + // Have to be careful not to take address of &a[0] if len == 0: + // pass nil in that case. + textcommon += fmt.Sprintf("\tvar _p%d *%s\n", n, m[1]) + textcommon += fmt.Sprintf("\tif len(%s) > 0 {\n\t\t_p%d = &%s[0]\n\t}\n", p.Name, n, p.Name) + argscommon = append(argscommon, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n), fmt.Sprintf("len(%s)", p.Name)) + argscall = append(argscall, fmt.Sprintf("_p%d uintptr", n), fmt.Sprintf("_lenp%d int", n)) + argsgc = append(argsgc, fmt.Sprintf("_p%d", n), fmt.Sprintf("uintptr(_lenp%d)", n)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(_p%d)", n), fmt.Sprintf("C.size_t(_lenp%d)", n)) + n++ + } else if p.Type == "int64" && endianness != "" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses int64 with 32 bits mode. Case not yet implemented\n") + } else if p.Type == "bool" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses bool. Case not yet implemented\n") + } else if regexp.MustCompile(`^_`).FindStringSubmatch(p.Type) != nil || p.Type == "unsafe.Pointer" { + argscommon = append(argscommon, fmt.Sprintf("uintptr(%s)", p.Name)) + argscall = append(argscall, fmt.Sprintf("%s uintptr", p.Name)) + argsgc = append(argsgc, p.Name) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else if p.Type == "int" { + if (argN == 0 || argN == 2) && ((funct == "fcntl") || (funct == "FcntlInt") || (funct == "FcntlFlock")) { + // These fcntl arguments need to be uintptr to be able to call FcntlInt and FcntlFlock + argscommon = append(argscommon, fmt.Sprintf("uintptr(%s)", p.Name)) + argscall = append(argscall, fmt.Sprintf("%s uintptr", p.Name)) + argsgc = append(argsgc, p.Name) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + + } else { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s int", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.int(%s)", p.Name)) + } + } else if p.Type == "int32" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s int32", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.int(%s)", p.Name)) + } else if p.Type == "int64" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s int64", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.longlong(%s)", p.Name)) + } else if p.Type == "uint32" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s uint32", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uint(%s)", p.Name)) + } else if p.Type == "uint64" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s uint64", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.ulonglong(%s)", p.Name)) + } else if p.Type == "uintptr" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s uintptr", p.Name)) + argsgc = append(argsgc, p.Name) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else { + argscommon = append(argscommon, fmt.Sprintf("int(%s)", p.Name)) + argscall = append(argscall, fmt.Sprintf("%s int", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.int(%s)", p.Name)) + } + argN++ + } + nargs := len(argsgc) + + // COMMON function generation + argscommonlist := strings.Join(argscommon, ", ") + callcommon := fmt.Sprintf("call%s(%s)", sysname, argscommonlist) + ret := []string{"_", "_"} + body := "" + doErrno := false + for i := 0; i < len(out); i++ { + p := parseParam(out[i]) + reg := "" + if p.Name == "err" { + reg = "e1" + ret[1] = reg + doErrno = true + } else { + reg = "r0" + ret[0] = reg + } + if p.Type == "bool" { + reg = fmt.Sprintf("%s != 0", reg) + } + if reg != "e1" { + body += fmt.Sprintf("\t%s = %s(%s)\n", p.Name, p.Type, reg) + } + } + if ret[0] == "_" && ret[1] == "_" { + textcommon += fmt.Sprintf("\t%s\n", callcommon) + } else { + textcommon += fmt.Sprintf("\t%s, %s := %s\n", ret[0], ret[1], callcommon) + } + textcommon += body + + if doErrno { + textcommon += "\tif e1 != 0 {\n" + textcommon += "\t\terr = errnoErr(e1)\n" + textcommon += "\t}\n" + } + textcommon += "\treturn\n" + textcommon += "}\n" + + if onlyCommon { + continue + } + + // CALL Prototype + callProto := fmt.Sprintf("func call%s(%s) (r1 uintptr, e1 Errno) {\n", sysname, strings.Join(argscall, ", ")) + + // GC function generation + asm := "syscall6" + if nonblock != nil { + asm = "rawSyscall6" + } + + if len(argsgc) <= 6 { + for len(argsgc) < 6 { + argsgc = append(argsgc, "0") + } + } else { + fmt.Fprintf(os.Stderr, "%s: too many arguments to system call", funct) + os.Exit(1) + } + argsgclist := strings.Join(argsgc, ", ") + callgc := fmt.Sprintf("%s(uintptr(unsafe.Pointer(&%s)), %d, %s)", asm, sysvarname, nargs, argsgclist) + + textgc += callProto + textgc += fmt.Sprintf("\tr1, _, e1 = %s\n", callgc) + textgc += "\treturn\n}\n" + + // GCCGO function generation + argsgccgolist := strings.Join(argsgccgo, ", ") + var callgccgo string + if sysname == "select" { + // select is a keyword of Go. Its name is + // changed to c_select. + callgccgo = fmt.Sprintf("C.c_%s(%s)", sysname, argsgccgolist) + } else { + callgccgo = fmt.Sprintf("C.%s(%s)", sysname, argsgccgolist) + } + textgccgo += callProto + textgccgo += fmt.Sprintf("\tr1 = uintptr(%s)\n", callgccgo) + textgccgo += "\te1 = syscall.GetErrno()\n" + textgccgo += "\treturn\n}\n" + } + if err := s.Err(); err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + file.Close() + } + imp := "" + if pack != "unix" { + imp = "import \"golang.org/x/sys/unix\"\n" + + } + + // Print zsyscall_aix_ppc64.go + err := ioutil.WriteFile("zsyscall_aix_ppc64.go", + []byte(fmt.Sprintf(srcTemplate1, cmdLine(), buildTags(), pack, imp, textcommon)), + 0644) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + // Print zsyscall_aix_ppc64_gc.go + vardecls := "\t" + strings.Join(vars, ",\n\t") + vardecls += " syscallFunc" + err = ioutil.WriteFile("zsyscall_aix_ppc64_gc.go", + []byte(fmt.Sprintf(srcTemplate2, cmdLine(), buildTags(), pack, imp, dynimports, linknames, vardecls, textgc)), + 0644) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + // Print zsyscall_aix_ppc64_gccgo.go + err = ioutil.WriteFile("zsyscall_aix_ppc64_gccgo.go", + []byte(fmt.Sprintf(srcTemplate3, cmdLine(), buildTags(), pack, cExtern, imp, textgccgo)), + 0644) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } +} + +const srcTemplate1 = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package %s + +import ( + "unsafe" +) + + +%s + +%s +` +const srcTemplate2 = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s +// +build !gccgo + +package %s + +import ( + "unsafe" +) +%s +%s +%s +type syscallFunc uintptr + +var ( +%s +) + +// Implemented in runtime/syscall_aix.go. +func rawSyscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) +func syscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) + +%s +` +const srcTemplate3 = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s +// +build gccgo + +package %s + +%s +*/ +import "C" +import ( + "syscall" +) + + +%s + +%s +` diff --git a/vendor/golang.org/x/sys/unix/mksyscall_solaris.go b/vendor/golang.org/x/sys/unix/mksyscall_solaris.go new file mode 100644 index 0000000000..3d864738b6 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksyscall_solaris.go @@ -0,0 +1,335 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* + This program reads a file containing function prototypes + (like syscall_solaris.go) and generates system call bodies. + The prototypes are marked by lines beginning with "//sys" + and read like func declarations if //sys is replaced by func, but: + * The parameter lists must give a name for each argument. + This includes return parameters. + * The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + * If the return parameter is an error number, it must be named err. + * If go func name needs to be different than its libc name, + * or the function is not in libc, name could be specified + * at the end, after "=" sign, like + //sys getsockopt(s int, level int, name int, val uintptr, vallen *_Socklen) (err error) = libsocket.getsockopt +*/ + +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "regexp" + "strings" +) + +var ( + b32 = flag.Bool("b32", false, "32bit big-endian") + l32 = flag.Bool("l32", false, "32bit little-endian") + tags = flag.String("tags", "", "build tags") +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksyscall_solaris.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return *tags +} + +// Param is function parameter +type Param struct { + Name string + Type string +} + +// usage prints the program usage +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run mksyscall_solaris.go [-b32 | -l32] [-tags x,y] [file ...]\n") + os.Exit(1) +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []string { + list = strings.TrimSpace(list) + if list == "" { + return []string{} + } + return regexp.MustCompile(`\s*,\s*`).Split(list, -1) +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexp.MustCompile(`^(\S*) (\S*)$`).FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + return Param{ps[1], ps[2]} +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + endianness := "" + if *b32 { + endianness = "big-endian" + } else if *l32 { + endianness = "little-endian" + } + + pack := "" + text := "" + dynimports := "" + linknames := "" + var vars []string + for _, path := range flag.Args() { + file, err := os.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + t := s.Text() + t = strings.TrimSpace(t) + t = regexp.MustCompile(`\s+`).ReplaceAllString(t, ` `) + if p := regexp.MustCompile(`^package (\S+)$`).FindStringSubmatch(t); p != nil && pack == "" { + pack = p[1] + } + nonblock := regexp.MustCompile(`^\/\/sysnb `).FindStringSubmatch(t) + if regexp.MustCompile(`^\/\/sys `).FindStringSubmatch(t) == nil && nonblock == nil { + continue + } + + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, err error) + // Split into name, in params, out params. + f := regexp.MustCompile(`^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*(?:(\w*)\.)?(\w*))?$`).FindStringSubmatch(t) + if f == nil { + fmt.Fprintf(os.Stderr, "%s:%s\nmalformed //sys declaration\n", path, t) + os.Exit(1) + } + funct, inps, outps, modname, sysname := f[2], f[3], f[4], f[5], f[6] + + // Split argument lists on comma. + in := parseParamList(inps) + out := parseParamList(outps) + + inps = strings.Join(in, ", ") + outps = strings.Join(out, ", ") + + // Try in vain to keep people from editing this file. + // The theory is that they jump into the middle of the file + // without reading the header. + text += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + + // So file name. + if modname == "" { + modname = "libc" + } + + // System call name. + if sysname == "" { + sysname = funct + } + + // System call pointer variable name. + sysvarname := fmt.Sprintf("proc%s", sysname) + + strconvfunc := "BytePtrFromString" + strconvtype := "*byte" + + sysname = strings.ToLower(sysname) // All libc functions are lowercase. + + // Runtime import of function to allow cross-platform builds. + dynimports += fmt.Sprintf("//go:cgo_import_dynamic libc_%s %s \"%s.so\"\n", sysname, sysname, modname) + // Link symbol to proc address variable. + linknames += fmt.Sprintf("//go:linkname %s libc_%s\n", sysvarname, sysname) + // Library proc address variable. + vars = append(vars, sysvarname) + + // Go function header. + outlist := strings.Join(out, ", ") + if outlist != "" { + outlist = fmt.Sprintf(" (%s)", outlist) + } + if text != "" { + text += "\n" + } + text += fmt.Sprintf("func %s(%s)%s {\n", funct, strings.Join(in, ", "), outlist) + + // Check if err return available + errvar := "" + for _, param := range out { + p := parseParam(param) + if p.Type == "error" { + errvar = p.Name + continue + } + } + + // Prepare arguments to Syscall. + var args []string + n := 0 + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + args = append(args, "uintptr(unsafe.Pointer("+p.Name+"))") + } else if p.Type == "string" && errvar != "" { + text += fmt.Sprintf("\tvar _p%d %s\n", n, strconvtype) + text += fmt.Sprintf("\t_p%d, %s = %s(%s)\n", n, errvar, strconvfunc, p.Name) + text += fmt.Sprintf("\tif %s != nil {\n\t\treturn\n\t}\n", errvar) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + n++ + } else if p.Type == "string" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses string arguments, but has no error return\n") + text += fmt.Sprintf("\tvar _p%d %s\n", n, strconvtype) + text += fmt.Sprintf("\t_p%d, _ = %s(%s)\n", n, strconvfunc, p.Name) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + n++ + } else if s := regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type); s != nil { + // Convert slice into pointer, length. + // Have to be careful not to take address of &a[0] if len == 0: + // pass nil in that case. + text += fmt.Sprintf("\tvar _p%d *%s\n", n, s[1]) + text += fmt.Sprintf("\tif len(%s) > 0 {\n\t\t_p%d = &%s[0]\n\t}\n", p.Name, n, p.Name) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n), fmt.Sprintf("uintptr(len(%s))", p.Name)) + n++ + } else if p.Type == "int64" && endianness != "" { + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } + } else if p.Type == "bool" { + text += fmt.Sprintf("\tvar _p%d uint32\n", n) + text += fmt.Sprintf("\tif %s {\n\t\t_p%d = 1\n\t} else {\n\t\t_p%d = 0\n\t}\n", p.Name, n, n) + args = append(args, fmt.Sprintf("uintptr(_p%d)", n)) + n++ + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name)) + } + } + nargs := len(args) + + // Determine which form to use; pad args with zeros. + asm := "sysvicall6" + if nonblock != nil { + asm = "rawSysvicall6" + } + if len(args) <= 6 { + for len(args) < 6 { + args = append(args, "0") + } + } else { + fmt.Fprintf(os.Stderr, "%s: too many arguments to system call\n", path) + os.Exit(1) + } + + // Actual call. + arglist := strings.Join(args, ", ") + call := fmt.Sprintf("%s(uintptr(unsafe.Pointer(&%s)), %d, %s)", asm, sysvarname, nargs, arglist) + + // Assign return values. + body := "" + ret := []string{"_", "_", "_"} + doErrno := false + for i := 0; i < len(out); i++ { + p := parseParam(out[i]) + reg := "" + if p.Name == "err" { + reg = "e1" + ret[2] = reg + doErrno = true + } else { + reg = fmt.Sprintf("r%d", i) + ret[i] = reg + } + if p.Type == "bool" { + reg = fmt.Sprintf("%d != 0", reg) + } + if p.Type == "int64" && endianness != "" { + // 64-bit number in r1:r0 or r0:r1. + if i+2 > len(out) { + fmt.Fprintf(os.Stderr, "%s: not enough registers for int64 return\n", path) + os.Exit(1) + } + if endianness == "big-endian" { + reg = fmt.Sprintf("int64(r%d)<<32 | int64(r%d)", i, i+1) + } else { + reg = fmt.Sprintf("int64(r%d)<<32 | int64(r%d)", i+1, i) + } + ret[i] = fmt.Sprintf("r%d", i) + ret[i+1] = fmt.Sprintf("r%d", i+1) + } + if reg != "e1" { + body += fmt.Sprintf("\t%s = %s(%s)\n", p.Name, p.Type, reg) + } + } + if ret[0] == "_" && ret[1] == "_" && ret[2] == "_" { + text += fmt.Sprintf("\t%s\n", call) + } else { + text += fmt.Sprintf("\t%s, %s, %s := %s\n", ret[0], ret[1], ret[2], call) + } + text += body + + if doErrno { + text += "\tif e1 != 0 {\n" + text += "\t\terr = e1\n" + text += "\t}\n" + } + text += "\treturn\n" + text += "}\n" + } + if err := s.Err(); err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + file.Close() + } + imp := "" + if pack != "unix" { + imp = "import \"golang.org/x/sys/unix\"\n" + + } + vardecls := "\t" + strings.Join(vars, ",\n\t") + vardecls += " syscallFunc" + fmt.Printf(srcTemplate, cmdLine(), buildTags(), pack, imp, dynimports, linknames, vardecls, text) +} + +const srcTemplate = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package %s + +import ( + "syscall" + "unsafe" +) +%s +%s +%s +var ( +%s +) + +%s +` diff --git a/vendor/golang.org/x/sys/unix/mksysctl_openbsd.go b/vendor/golang.org/x/sys/unix/mksysctl_openbsd.go new file mode 100644 index 0000000000..b6b409909c --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksysctl_openbsd.go @@ -0,0 +1,355 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Parse the header files for OpenBSD and generate a Go usable sysctl MIB. +// +// Build a MIB with each entry being an array containing the level, type and +// a hash that will contain additional entries if the current entry is a node. +// We then walk this MIB and create a flattened sysctl name to OID hash. + +package main + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +var ( + goos, goarch string +) + +// cmdLine returns this programs's commandline arguments. +func cmdLine() string { + return "go run mksysctl_openbsd.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags. +func buildTags() string { + return fmt.Sprintf("%s,%s", goarch, goos) +} + +// reMatch performs regular expression match and stores the substring slice to value pointed by m. +func reMatch(re *regexp.Regexp, str string, m *[]string) bool { + *m = re.FindStringSubmatch(str) + if *m != nil { + return true + } + return false +} + +type nodeElement struct { + n int + t string + pE *map[string]nodeElement +} + +var ( + debugEnabled bool + mib map[string]nodeElement + node *map[string]nodeElement + nodeMap map[string]string + sysCtl []string +) + +var ( + ctlNames1RE = regexp.MustCompile(`^#define\s+(CTL_NAMES)\s+{`) + ctlNames2RE = regexp.MustCompile(`^#define\s+(CTL_(.*)_NAMES)\s+{`) + ctlNames3RE = regexp.MustCompile(`^#define\s+((.*)CTL_NAMES)\s+{`) + netInetRE = regexp.MustCompile(`^netinet/`) + netInet6RE = regexp.MustCompile(`^netinet6/`) + netRE = regexp.MustCompile(`^net/`) + bracesRE = regexp.MustCompile(`{.*}`) + ctlTypeRE = regexp.MustCompile(`{\s+"(\w+)",\s+(CTLTYPE_[A-Z]+)\s+}`) + fsNetKernRE = regexp.MustCompile(`^(fs|net|kern)_`) +) + +func debug(s string) { + if debugEnabled { + fmt.Fprintln(os.Stderr, s) + } +} + +// Walk the MIB and build a sysctl name to OID mapping. +func buildSysctl(pNode *map[string]nodeElement, name string, oid []int) { + lNode := pNode // local copy of pointer to node + var keys []string + for k := range *lNode { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, key := range keys { + nodename := name + if name != "" { + nodename += "." + } + nodename += key + + nodeoid := append(oid, (*pNode)[key].n) + + if (*pNode)[key].t == `CTLTYPE_NODE` { + if _, ok := nodeMap[nodename]; ok { + lNode = &mib + ctlName := nodeMap[nodename] + for _, part := range strings.Split(ctlName, ".") { + lNode = ((*lNode)[part]).pE + } + } else { + lNode = (*pNode)[key].pE + } + buildSysctl(lNode, nodename, nodeoid) + } else if (*pNode)[key].t != "" { + oidStr := []string{} + for j := range nodeoid { + oidStr = append(oidStr, fmt.Sprintf("%d", nodeoid[j])) + } + text := "\t{ \"" + nodename + "\", []_C_int{ " + strings.Join(oidStr, ", ") + " } }, \n" + sysCtl = append(sysCtl, text) + } + } +} + +func main() { + // Get the OS (using GOOS_TARGET if it exist) + goos = os.Getenv("GOOS_TARGET") + if goos == "" { + goos = os.Getenv("GOOS") + } + // Get the architecture (using GOARCH_TARGET if it exists) + goarch = os.Getenv("GOARCH_TARGET") + if goarch == "" { + goarch = os.Getenv("GOARCH") + } + // Check if GOOS and GOARCH environment variables are defined + if goarch == "" || goos == "" { + fmt.Fprintf(os.Stderr, "GOARCH or GOOS not defined in environment\n") + os.Exit(1) + } + + mib = make(map[string]nodeElement) + headers := [...]string{ + `sys/sysctl.h`, + `sys/socket.h`, + `sys/tty.h`, + `sys/malloc.h`, + `sys/mount.h`, + `sys/namei.h`, + `sys/sem.h`, + `sys/shm.h`, + `sys/vmmeter.h`, + `uvm/uvmexp.h`, + `uvm/uvm_param.h`, + `uvm/uvm_swap_encrypt.h`, + `ddb/db_var.h`, + `net/if.h`, + `net/if_pfsync.h`, + `net/pipex.h`, + `netinet/in.h`, + `netinet/icmp_var.h`, + `netinet/igmp_var.h`, + `netinet/ip_ah.h`, + `netinet/ip_carp.h`, + `netinet/ip_divert.h`, + `netinet/ip_esp.h`, + `netinet/ip_ether.h`, + `netinet/ip_gre.h`, + `netinet/ip_ipcomp.h`, + `netinet/ip_ipip.h`, + `netinet/pim_var.h`, + `netinet/tcp_var.h`, + `netinet/udp_var.h`, + `netinet6/in6.h`, + `netinet6/ip6_divert.h`, + `netinet6/pim6_var.h`, + `netinet/icmp6.h`, + `netmpls/mpls.h`, + } + + ctls := [...]string{ + `kern`, + `vm`, + `fs`, + `net`, + //debug /* Special handling required */ + `hw`, + //machdep /* Arch specific */ + `user`, + `ddb`, + //vfs /* Special handling required */ + `fs.posix`, + `kern.forkstat`, + `kern.intrcnt`, + `kern.malloc`, + `kern.nchstats`, + `kern.seminfo`, + `kern.shminfo`, + `kern.timecounter`, + `kern.tty`, + `kern.watchdog`, + `net.bpf`, + `net.ifq`, + `net.inet`, + `net.inet.ah`, + `net.inet.carp`, + `net.inet.divert`, + `net.inet.esp`, + `net.inet.etherip`, + `net.inet.gre`, + `net.inet.icmp`, + `net.inet.igmp`, + `net.inet.ip`, + `net.inet.ip.ifq`, + `net.inet.ipcomp`, + `net.inet.ipip`, + `net.inet.mobileip`, + `net.inet.pfsync`, + `net.inet.pim`, + `net.inet.tcp`, + `net.inet.udp`, + `net.inet6`, + `net.inet6.divert`, + `net.inet6.ip6`, + `net.inet6.icmp6`, + `net.inet6.pim6`, + `net.inet6.tcp6`, + `net.inet6.udp6`, + `net.mpls`, + `net.mpls.ifq`, + `net.key`, + `net.pflow`, + `net.pfsync`, + `net.pipex`, + `net.rt`, + `vm.swapencrypt`, + //vfsgenctl /* Special handling required */ + } + + // Node name "fixups" + ctlMap := map[string]string{ + "ipproto": "net.inet", + "net.inet.ipproto": "net.inet", + "net.inet6.ipv6proto": "net.inet6", + "net.inet6.ipv6": "net.inet6.ip6", + "net.inet.icmpv6": "net.inet6.icmp6", + "net.inet6.divert6": "net.inet6.divert", + "net.inet6.tcp6": "net.inet.tcp", + "net.inet6.udp6": "net.inet.udp", + "mpls": "net.mpls", + "swpenc": "vm.swapencrypt", + } + + // Node mappings + nodeMap = map[string]string{ + "net.inet.ip.ifq": "net.ifq", + "net.inet.pfsync": "net.pfsync", + "net.mpls.ifq": "net.ifq", + } + + mCtls := make(map[string]bool) + for _, ctl := range ctls { + mCtls[ctl] = true + } + + for _, header := range headers { + debug("Processing " + header) + file, err := os.Open(filepath.Join("/usr/include", header)) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + var sub []string + if reMatch(ctlNames1RE, s.Text(), &sub) || + reMatch(ctlNames2RE, s.Text(), &sub) || + reMatch(ctlNames3RE, s.Text(), &sub) { + if sub[1] == `CTL_NAMES` { + // Top level. + node = &mib + } else { + // Node. + nodename := strings.ToLower(sub[2]) + ctlName := "" + if reMatch(netInetRE, header, &sub) { + ctlName = "net.inet." + nodename + } else if reMatch(netInet6RE, header, &sub) { + ctlName = "net.inet6." + nodename + } else if reMatch(netRE, header, &sub) { + ctlName = "net." + nodename + } else { + ctlName = nodename + ctlName = fsNetKernRE.ReplaceAllString(ctlName, `$1.`) + } + + if val, ok := ctlMap[ctlName]; ok { + ctlName = val + } + if _, ok := mCtls[ctlName]; !ok { + debug("Ignoring " + ctlName + "...") + continue + } + + // Walk down from the top of the MIB. + node = &mib + for _, part := range strings.Split(ctlName, ".") { + if _, ok := (*node)[part]; !ok { + debug("Missing node " + part) + (*node)[part] = nodeElement{n: 0, t: "", pE: &map[string]nodeElement{}} + } + node = (*node)[part].pE + } + } + + // Populate current node with entries. + i := -1 + for !strings.HasPrefix(s.Text(), "}") { + s.Scan() + if reMatch(bracesRE, s.Text(), &sub) { + i++ + } + if !reMatch(ctlTypeRE, s.Text(), &sub) { + continue + } + (*node)[sub[1]] = nodeElement{n: i, t: sub[2], pE: &map[string]nodeElement{}} + } + } + } + err = s.Err() + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + file.Close() + } + buildSysctl(&mib, "", []int{}) + + sort.Strings(sysCtl) + text := strings.Join(sysCtl, "") + + fmt.Printf(srcTemplate, cmdLine(), buildTags(), text) +} + +const srcTemplate = `// %s +// Code generated by the command above; DO NOT EDIT. + +// +build %s + +package unix + +type mibentry struct { + ctlname string + ctloid []_C_int +} + +var sysctlMib = []mibentry { +%s +} +` diff --git a/vendor/golang.org/x/sys/unix/mksysnum.go b/vendor/golang.org/x/sys/unix/mksysnum.go new file mode 100644 index 0000000000..baa6ecd850 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksysnum.go @@ -0,0 +1,190 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Generate system call table for DragonFly, NetBSD, +// FreeBSD, OpenBSD or Darwin from master list +// (for example, /usr/src/sys/kern/syscalls.master or +// sys/syscall.h). +package main + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "regexp" + "strings" +) + +var ( + goos, goarch string +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksysnum.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return fmt.Sprintf("%s,%s", goarch, goos) +} + +func checkErr(err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +// source string and substring slice for regexp +type re struct { + str string // source string + sub []string // matched sub-string +} + +// Match performs regular expression match +func (r *re) Match(exp string) bool { + r.sub = regexp.MustCompile(exp).FindStringSubmatch(r.str) + if r.sub != nil { + return true + } + return false +} + +// fetchFile fetches a text file from URL +func fetchFile(URL string) io.Reader { + resp, err := http.Get(URL) + checkErr(err) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + checkErr(err) + return strings.NewReader(string(body)) +} + +// readFile reads a text file from path +func readFile(path string) io.Reader { + file, err := os.Open(os.Args[1]) + checkErr(err) + return file +} + +func format(name, num, proto string) string { + name = strings.ToUpper(name) + // There are multiple entries for enosys and nosys, so comment them out. + nm := re{str: name} + if nm.Match(`^SYS_E?NOSYS$`) { + name = fmt.Sprintf("// %s", name) + } + if name == `SYS_SYS_EXIT` { + name = `SYS_EXIT` + } + return fmt.Sprintf(" %s = %s; // %s\n", name, num, proto) +} + +func main() { + // Get the OS (using GOOS_TARGET if it exist) + goos = os.Getenv("GOOS_TARGET") + if goos == "" { + goos = os.Getenv("GOOS") + } + // Get the architecture (using GOARCH_TARGET if it exists) + goarch = os.Getenv("GOARCH_TARGET") + if goarch == "" { + goarch = os.Getenv("GOARCH") + } + // Check if GOOS and GOARCH environment variables are defined + if goarch == "" || goos == "" { + fmt.Fprintf(os.Stderr, "GOARCH or GOOS not defined in environment\n") + os.Exit(1) + } + + file := strings.TrimSpace(os.Args[1]) + var syscalls io.Reader + if strings.HasPrefix(file, "https://") || strings.HasPrefix(file, "http://") { + // Download syscalls.master file + syscalls = fetchFile(file) + } else { + syscalls = readFile(file) + } + + var text, line string + s := bufio.NewScanner(syscalls) + for s.Scan() { + t := re{str: line} + if t.Match(`^(.*)\\$`) { + // Handle continuation + line = t.sub[1] + line += strings.TrimLeft(s.Text(), " \t") + } else { + // New line + line = s.Text() + } + t = re{str: line} + if t.Match(`\\$`) { + continue + } + t = re{str: line} + + switch goos { + case "dragonfly": + if t.Match(`^([0-9]+)\s+STD\s+({ \S+\s+(\w+).*)$`) { + num, proto := t.sub[1], t.sub[2] + name := fmt.Sprintf("SYS_%s", t.sub[3]) + text += format(name, num, proto) + } + case "freebsd": + if t.Match(`^([0-9]+)\s+\S+\s+(?:(?:NO)?STD|COMPAT10)\s+({ \S+\s+(\w+).*)$`) { + num, proto := t.sub[1], t.sub[2] + name := fmt.Sprintf("SYS_%s", t.sub[3]) + text += format(name, num, proto) + } + case "openbsd": + if t.Match(`^([0-9]+)\s+STD\s+(NOLOCK\s+)?({ \S+\s+\*?(\w+).*)$`) { + num, proto, name := t.sub[1], t.sub[3], t.sub[4] + text += format(name, num, proto) + } + case "netbsd": + if t.Match(`^([0-9]+)\s+((STD)|(NOERR))\s+(RUMP\s+)?({\s+\S+\s*\*?\s*\|(\S+)\|(\S*)\|(\w+).*\s+})(\s+(\S+))?$`) { + num, proto, compat := t.sub[1], t.sub[6], t.sub[8] + name := t.sub[7] + "_" + t.sub[9] + if t.sub[11] != "" { + name = t.sub[7] + "_" + t.sub[11] + } + name = strings.ToUpper(name) + if compat == "" || compat == "13" || compat == "30" || compat == "50" { + text += fmt.Sprintf(" %s = %s; // %s\n", name, num, proto) + } + } + case "darwin": + if t.Match(`^#define\s+SYS_(\w+)\s+([0-9]+)`) { + name, num := t.sub[1], t.sub[2] + name = strings.ToUpper(name) + text += fmt.Sprintf(" SYS_%s = %s;\n", name, num) + } + default: + fmt.Fprintf(os.Stderr, "unrecognized GOOS=%s\n", goos) + os.Exit(1) + + } + } + err := s.Err() + checkErr(err) + + fmt.Printf(template, cmdLine(), buildTags(), text) +} + +const template = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package unix + +const( +%s)` diff --git a/vendor/golang.org/x/sys/unix/types_aix.go b/vendor/golang.org/x/sys/unix/types_aix.go new file mode 100644 index 0000000000..40d2beede5 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_aix.go @@ -0,0 +1,237 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore +// +build aix + +/* +Input to cgo -godefs. See also mkerrors.sh and mkall.sh +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + + +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong + PathMax = C.PATH_MAX +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +type off64 C.off64_t +type off C.off_t +type Mode_t C.mode_t + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +type Timeval32 C.struct_timeval32 + +type Timex C.struct_timex + +type Time_t C.time_t + +type Tms C.struct_tms + +type Utimbuf C.struct_utimbuf + +type Timezone C.struct_timezone + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit64 + +type Pid_t C.pid_t + +type _Gid_t C.gid_t + +type dev_t C.dev_t + +// Files + +type Stat_t C.struct_stat + +type StatxTimestamp C.struct_statx_timestamp + +type Statx_t C.struct_statx + +type Dirent C.struct_dirent + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Cmsghdr C.struct_cmsghdr + +type ICMPv6Filter C.struct_icmp6_filter + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type Linger C.struct_linger + +type Msghdr C.struct_msghdr + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr +) + +type IfMsgHdr C.struct_if_msghdr + +// Misc + +type FdSet C.fd_set + +type Utsname C.struct_utsname + +type Ustat_t C.struct_ustat + +type Sigset_t C.sigset_t + +const ( + AT_FDCWD = C.AT_FDCWD + AT_REMOVEDIR = C.AT_REMOVEDIR + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// Terminal handling + +type Termios C.struct_termios + +type Termio C.struct_termio + +type Winsize C.struct_winsize + +//poll + +type PollFd struct { + Fd int32 + Events uint16 + Revents uint16 +} + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +//flock_t + +type Flock_t C.struct_flock64 + +// Statfs + +type Fsid_t C.struct_fsid_t +type Fsid64_t C.struct_fsid64_t + +type Statfs_t C.struct_statfs + +const RNDGETENTCNT = 0x80045200 diff --git a/vendor/golang.org/x/sys/unix/types_darwin.go b/vendor/golang.org/x/sys/unix/types_darwin.go new file mode 100644 index 0000000000..155c2e692b --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_darwin.go @@ -0,0 +1,283 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define __DARWIN_UNIX03 0 +#define KERNEL +#define _DARWIN_USE_64_BIT_INODE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +type Timeval32 C.struct_timeval32 + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat64 + +type Statfs_t C.struct_statfs64 + +type Flock_t C.struct_flock + +type Fstore_t C.struct_fstore + +type Radvisory_t C.struct_radvisory + +type Fbootstraptransfer_t C.struct_fbootstraptransfer + +type Log2phys_t C.struct_log2phys + +type Fsid C.struct_fsid + +type Dirent C.struct_dirent + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet4Pktinfo C.struct_in_pktinfo + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet4Pktinfo = C.sizeof_struct_in_pktinfo + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr + SizeofIfmaMsghdr2 = C.sizeof_struct_ifma_msghdr2 + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type IfmaMsghdr C.struct_ifma_msghdr + +type IfmaMsghdr2 C.struct_ifma_msghdr2 + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_REMOVEDIR = C.AT_REMOVEDIR + AT_SYMLINK_FOLLOW = C.AT_SYMLINK_FOLLOW + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// uname + +type Utsname C.struct_utsname + +// Clockinfo + +const SizeofClockinfo = C.sizeof_struct_clockinfo + +type Clockinfo C.struct_clockinfo diff --git a/vendor/golang.org/x/sys/unix/types_dragonfly.go b/vendor/golang.org/x/sys/unix/types_dragonfly.go new file mode 100644 index 0000000000..3365dd79d0 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_dragonfly.go @@ -0,0 +1,263 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define KERNEL +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat + +type Statfs_t C.struct_statfs + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +type Fsid C.struct_fsid + +// File system limits + +const ( + PathMax = C.PATH_MAX +) + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr + SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type IfmaMsghdr C.struct_ifma_msghdr + +type IfAnnounceMsghdr C.struct_if_announcemsghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// Uname + +type Utsname C.struct_utsname diff --git a/vendor/golang.org/x/sys/unix/types_freebsd.go b/vendor/golang.org/x/sys/unix/types_freebsd.go new file mode 100644 index 0000000000..7470798951 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_freebsd.go @@ -0,0 +1,356 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define _WANT_FREEBSD11_STAT 1 +#define _WANT_FREEBSD11_STATFS 1 +#define _WANT_FREEBSD11_DIRENT 1 +#define _WANT_FREEBSD11_KEVENT 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +// This structure is a duplicate of if_data on FreeBSD 8-STABLE. +// See /usr/include/net/if.h. +struct if_data8 { + u_char ifi_type; + u_char ifi_physical; + u_char ifi_addrlen; + u_char ifi_hdrlen; + u_char ifi_link_state; + u_char ifi_spare_char1; + u_char ifi_spare_char2; + u_char ifi_datalen; + u_long ifi_mtu; + u_long ifi_metric; + u_long ifi_baudrate; + u_long ifi_ipackets; + u_long ifi_ierrors; + u_long ifi_opackets; + u_long ifi_oerrors; + u_long ifi_collisions; + u_long ifi_ibytes; + u_long ifi_obytes; + u_long ifi_imcasts; + u_long ifi_omcasts; + u_long ifi_iqdrops; + u_long ifi_noproto; + u_long ifi_hwassist; +// FIXME: these are now unions, so maybe need to change definitions? +#undef ifi_epoch + time_t ifi_epoch; +#undef ifi_lastchange + struct timeval ifi_lastchange; +}; + +// This structure is a duplicate of if_msghdr on FreeBSD 8-STABLE. +// See /usr/include/net/if.h. +struct if_msghdr8 { + u_short ifm_msglen; + u_char ifm_version; + u_char ifm_type; + int ifm_addrs; + int ifm_flags; + u_short ifm_index; + struct if_data8 ifm_data; +}; +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +const ( + _statfsVersion = C.STATFS_VERSION + _dirblksiz = C.DIRBLKSIZ +) + +type Stat_t C.struct_stat + +type stat_freebsd11_t C.struct_freebsd11_stat + +type Statfs_t C.struct_statfs + +type statfs_freebsd11_t C.struct_freebsd11_statfs + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +type dirent_freebsd11 C.struct_freebsd11_dirent + +type Fsid C.struct_fsid + +// File system limits + +const ( + PathMax = C.PATH_MAX +) + +// Advice to Fadvise + +const ( + FADV_NORMAL = C.POSIX_FADV_NORMAL + FADV_RANDOM = C.POSIX_FADV_RANDOM + FADV_SEQUENTIAL = C.POSIX_FADV_SEQUENTIAL + FADV_WILLNEED = C.POSIX_FADV_WILLNEED + FADV_DONTNEED = C.POSIX_FADV_DONTNEED + FADV_NOREUSE = C.POSIX_FADV_NOREUSE +) + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPMreqn C.struct_ip_mreqn + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPMreqn = C.sizeof_struct_ip_mreqn + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent_freebsd11 + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + sizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfMsghdr = C.sizeof_struct_if_msghdr8 + sizeofIfData = C.sizeof_struct_if_data + SizeofIfData = C.sizeof_struct_if_data8 + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr + SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type ifMsghdr C.struct_if_msghdr + +type IfMsghdr C.struct_if_msghdr8 + +type ifData C.struct_if_data + +type IfData C.struct_if_data8 + +type IfaMsghdr C.struct_ifa_msghdr + +type IfmaMsghdr C.struct_ifma_msghdr + +type IfAnnounceMsghdr C.struct_if_announcemsghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfZbuf = C.sizeof_struct_bpf_zbuf + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr + SizeofBpfZbufHeader = C.sizeof_struct_bpf_zbuf_header +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfZbuf C.struct_bpf_zbuf + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +type BpfZbufHeader C.struct_bpf_zbuf_header + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_REMOVEDIR = C.AT_REMOVEDIR + AT_SYMLINK_FOLLOW = C.AT_SYMLINK_FOLLOW + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLINIGNEOF = C.POLLINIGNEOF + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// Capabilities + +type CapRights C.struct_cap_rights + +// Uname + +type Utsname C.struct_utsname diff --git a/vendor/golang.org/x/sys/unix/types_netbsd.go b/vendor/golang.org/x/sys/unix/types_netbsd.go new file mode 100644 index 0000000000..2dd4f9542c --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_netbsd.go @@ -0,0 +1,289 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define KERNEL +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat + +type Statfs_t C.struct_statfs + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +type Fsid C.fsid_t + +// File system limits + +const ( + PathMax = C.PATH_MAX +) + +// Advice to Fadvise + +const ( + FADV_NORMAL = C.POSIX_FADV_NORMAL + FADV_RANDOM = C.POSIX_FADV_RANDOM + FADV_SEQUENTIAL = C.POSIX_FADV_SEQUENTIAL + FADV_WILLNEED = C.POSIX_FADV_WILLNEED + FADV_DONTNEED = C.POSIX_FADV_DONTNEED + FADV_NOREUSE = C.POSIX_FADV_NOREUSE +) + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type IfAnnounceMsghdr C.struct_if_announcemsghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +type Mclpool C.struct_mclpool + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +type BpfTimeval C.struct_bpf_timeval + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +type Ptmget C.struct_ptmget + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// Sysctl + +type Sysctlnode C.struct_sysctlnode + +// Uname + +type Utsname C.struct_utsname + +// Clockinfo + +const SizeofClockinfo = C.sizeof_struct_clockinfo + +type Clockinfo C.struct_clockinfo diff --git a/vendor/golang.org/x/sys/unix/types_openbsd.go b/vendor/golang.org/x/sys/unix/types_openbsd.go new file mode 100644 index 0000000000..8aafbe4469 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_openbsd.go @@ -0,0 +1,282 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define KERNEL +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat + +type Statfs_t C.struct_statfs + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +type Fsid C.fsid_t + +// File system limits + +const ( + PathMax = C.PATH_MAX +) + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type IfAnnounceMsghdr C.struct_if_announcemsghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +type Mclpool C.struct_mclpool + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +type BpfTimeval C.struct_bpf_timeval + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// Signal Sets + +type Sigset_t C.sigset_t + +// Uname + +type Utsname C.struct_utsname + +// Uvmexp + +const SizeofUvmexp = C.sizeof_struct_uvmexp + +type Uvmexp C.struct_uvmexp + +// Clockinfo + +const SizeofClockinfo = C.sizeof_struct_clockinfo + +type Clockinfo C.struct_clockinfo diff --git a/vendor/golang.org/x/sys/unix/types_solaris.go b/vendor/golang.org/x/sys/unix/types_solaris.go new file mode 100644 index 0000000000..2b716f9348 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_solaris.go @@ -0,0 +1,266 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define KERNEL +// These defines ensure that builds done on newer versions of Solaris are +// backwards-compatible with older versions of Solaris and +// OpenSolaris-based derivatives. +#define __USE_SUNOS_SOCKETS__ // msghdr +#define __USE_LEGACY_PROTOTYPES__ // iovec +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong + PathMax = C.PATH_MAX + MaxHostNameLen = C.MAXHOSTNAMELEN +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +type Timeval32 C.struct_timeval32 + +type Tms C.struct_tms + +type Utimbuf C.struct_utimbuf + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +// Filesystems + +type _Fsblkcnt_t C.fsblkcnt_t + +type Statvfs_t C.struct_statvfs + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Select + +type FdSet C.fd_set + +// Misc + +type Utsname C.struct_utsname + +type Ustat_t C.struct_ustat + +const ( + AT_FDCWD = C.AT_FDCWD + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW + AT_SYMLINK_FOLLOW = C.AT_SYMLINK_FOLLOW + AT_REMOVEDIR = C.AT_REMOVEDIR + AT_EACCESS = C.AT_EACCESS +) + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfTimeval C.struct_bpf_timeval + +type BpfHdr C.struct_bpf_hdr + +// Terminal handling + +type Termios C.struct_termios + +type Termio C.struct_termio + +type Winsize C.struct_winsize + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) diff --git a/vendor/golang.org/x/text/encoding/charmap/maketables.go b/vendor/golang.org/x/text/encoding/charmap/maketables.go new file mode 100644 index 0000000000..f7941701e8 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/charmap/maketables.go @@ -0,0 +1,556 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" + "unicode/utf8" + + "golang.org/x/text/encoding" + "golang.org/x/text/internal/gen" +) + +const ascii = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !"#$%&'()*+,-./0123456789:;<=>?` + + `@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_` + + "`abcdefghijklmnopqrstuvwxyz{|}~\u007f" + +var encodings = []struct { + name string + mib string + comment string + varName string + replacement byte + mapping string +}{ + { + "IBM Code Page 037", + "IBM037", + "", + "CodePage037", + 0x3f, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM037-2.1.2.ucm", + }, + { + "IBM Code Page 437", + "PC8CodePage437", + "", + "CodePage437", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM437-2.1.2.ucm", + }, + { + "IBM Code Page 850", + "PC850Multilingual", + "", + "CodePage850", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM850-2.1.2.ucm", + }, + { + "IBM Code Page 852", + "PCp852", + "", + "CodePage852", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM852-2.1.2.ucm", + }, + { + "IBM Code Page 855", + "IBM855", + "", + "CodePage855", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM855-2.1.2.ucm", + }, + { + "Windows Code Page 858", // PC latin1 with Euro + "IBM00858", + "", + "CodePage858", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/windows-858-2000.ucm", + }, + { + "IBM Code Page 860", + "IBM860", + "", + "CodePage860", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM860-2.1.2.ucm", + }, + { + "IBM Code Page 862", + "PC862LatinHebrew", + "", + "CodePage862", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM862-2.1.2.ucm", + }, + { + "IBM Code Page 863", + "IBM863", + "", + "CodePage863", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM863-2.1.2.ucm", + }, + { + "IBM Code Page 865", + "IBM865", + "", + "CodePage865", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM865-2.1.2.ucm", + }, + { + "IBM Code Page 866", + "IBM866", + "", + "CodePage866", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-ibm866.txt", + }, + { + "IBM Code Page 1047", + "IBM1047", + "", + "CodePage1047", + 0x3f, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM1047-2.1.2.ucm", + }, + { + "IBM Code Page 1140", + "IBM01140", + "", + "CodePage1140", + 0x3f, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/ibm-1140_P100-1997.ucm", + }, + { + "ISO 8859-1", + "ISOLatin1", + "", + "ISO8859_1", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/iso-8859_1-1998.ucm", + }, + { + "ISO 8859-2", + "ISOLatin2", + "", + "ISO8859_2", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-2.txt", + }, + { + "ISO 8859-3", + "ISOLatin3", + "", + "ISO8859_3", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-3.txt", + }, + { + "ISO 8859-4", + "ISOLatin4", + "", + "ISO8859_4", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-4.txt", + }, + { + "ISO 8859-5", + "ISOLatinCyrillic", + "", + "ISO8859_5", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-5.txt", + }, + { + "ISO 8859-6", + "ISOLatinArabic", + "", + "ISO8859_6,ISO8859_6E,ISO8859_6I", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-6.txt", + }, + { + "ISO 8859-7", + "ISOLatinGreek", + "", + "ISO8859_7", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-7.txt", + }, + { + "ISO 8859-8", + "ISOLatinHebrew", + "", + "ISO8859_8,ISO8859_8E,ISO8859_8I", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-8.txt", + }, + { + "ISO 8859-9", + "ISOLatin5", + "", + "ISO8859_9", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/iso-8859_9-1999.ucm", + }, + { + "ISO 8859-10", + "ISOLatin6", + "", + "ISO8859_10", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-10.txt", + }, + { + "ISO 8859-13", + "ISO885913", + "", + "ISO8859_13", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-13.txt", + }, + { + "ISO 8859-14", + "ISO885914", + "", + "ISO8859_14", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-14.txt", + }, + { + "ISO 8859-15", + "ISO885915", + "", + "ISO8859_15", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-15.txt", + }, + { + "ISO 8859-16", + "ISO885916", + "", + "ISO8859_16", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-16.txt", + }, + { + "KOI8-R", + "KOI8R", + "", + "KOI8R", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-koi8-r.txt", + }, + { + "KOI8-U", + "KOI8U", + "", + "KOI8U", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-koi8-u.txt", + }, + { + "Macintosh", + "Macintosh", + "", + "Macintosh", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-macintosh.txt", + }, + { + "Macintosh Cyrillic", + "MacintoshCyrillic", + "", + "MacintoshCyrillic", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-x-mac-cyrillic.txt", + }, + { + "Windows 874", + "Windows874", + "", + "Windows874", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-874.txt", + }, + { + "Windows 1250", + "Windows1250", + "", + "Windows1250", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1250.txt", + }, + { + "Windows 1251", + "Windows1251", + "", + "Windows1251", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1251.txt", + }, + { + "Windows 1252", + "Windows1252", + "", + "Windows1252", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1252.txt", + }, + { + "Windows 1253", + "Windows1253", + "", + "Windows1253", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1253.txt", + }, + { + "Windows 1254", + "Windows1254", + "", + "Windows1254", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1254.txt", + }, + { + "Windows 1255", + "Windows1255", + "", + "Windows1255", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1255.txt", + }, + { + "Windows 1256", + "Windows1256", + "", + "Windows1256", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1256.txt", + }, + { + "Windows 1257", + "Windows1257", + "", + "Windows1257", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1257.txt", + }, + { + "Windows 1258", + "Windows1258", + "", + "Windows1258", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1258.txt", + }, + { + "X-User-Defined", + "XUserDefined", + "It is defined at http://encoding.spec.whatwg.org/#x-user-defined", + "XUserDefined", + encoding.ASCIISub, + ascii + + "\uf780\uf781\uf782\uf783\uf784\uf785\uf786\uf787" + + "\uf788\uf789\uf78a\uf78b\uf78c\uf78d\uf78e\uf78f" + + "\uf790\uf791\uf792\uf793\uf794\uf795\uf796\uf797" + + "\uf798\uf799\uf79a\uf79b\uf79c\uf79d\uf79e\uf79f" + + "\uf7a0\uf7a1\uf7a2\uf7a3\uf7a4\uf7a5\uf7a6\uf7a7" + + "\uf7a8\uf7a9\uf7aa\uf7ab\uf7ac\uf7ad\uf7ae\uf7af" + + "\uf7b0\uf7b1\uf7b2\uf7b3\uf7b4\uf7b5\uf7b6\uf7b7" + + "\uf7b8\uf7b9\uf7ba\uf7bb\uf7bc\uf7bd\uf7be\uf7bf" + + "\uf7c0\uf7c1\uf7c2\uf7c3\uf7c4\uf7c5\uf7c6\uf7c7" + + "\uf7c8\uf7c9\uf7ca\uf7cb\uf7cc\uf7cd\uf7ce\uf7cf" + + "\uf7d0\uf7d1\uf7d2\uf7d3\uf7d4\uf7d5\uf7d6\uf7d7" + + "\uf7d8\uf7d9\uf7da\uf7db\uf7dc\uf7dd\uf7de\uf7df" + + "\uf7e0\uf7e1\uf7e2\uf7e3\uf7e4\uf7e5\uf7e6\uf7e7" + + "\uf7e8\uf7e9\uf7ea\uf7eb\uf7ec\uf7ed\uf7ee\uf7ef" + + "\uf7f0\uf7f1\uf7f2\uf7f3\uf7f4\uf7f5\uf7f6\uf7f7" + + "\uf7f8\uf7f9\uf7fa\uf7fb\uf7fc\uf7fd\uf7fe\uf7ff", + }, +} + +func getWHATWG(url string) string { + res, err := http.Get(url) + if err != nil { + log.Fatalf("%q: Get: %v", url, err) + } + defer res.Body.Close() + + mapping := make([]rune, 128) + for i := range mapping { + mapping[i] = '\ufffd' + } + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := 0, 0 + if _, err := fmt.Sscanf(s, "%d\t0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0 || 128 <= x { + log.Fatalf("code %d is out of range", x) + } + if 0x80 <= y && y < 0xa0 { + // We diverge from the WHATWG spec by mapping control characters + // in the range [0x80, 0xa0) to U+FFFD. + continue + } + mapping[x] = rune(y) + } + return ascii + string(mapping) +} + +func getUCM(url string) string { + res, err := http.Get(url) + if err != nil { + log.Fatalf("%q: Get: %v", url, err) + } + defer res.Body.Close() + + mapping := make([]rune, 256) + for i := range mapping { + mapping[i] = '\ufffd' + } + + charsFound := 0 + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + var c byte + var r rune + if _, err := fmt.Sscanf(s, ` \x%x |0`, &r, &c); err != nil { + continue + } + mapping[c] = r + charsFound++ + } + + if charsFound < 200 { + log.Fatalf("%q: only %d characters found (wrong page format?)", url, charsFound) + } + + return string(mapping) +} + +func main() { + mibs := map[string]bool{} + all := []string{} + + w := gen.NewCodeWriter() + defer w.WriteGoFile("tables.go", "charmap") + + printf := func(s string, a ...interface{}) { fmt.Fprintf(w, s, a...) } + + printf("import (\n") + printf("\t\"golang.org/x/text/encoding\"\n") + printf("\t\"golang.org/x/text/encoding/internal/identifier\"\n") + printf(")\n\n") + for _, e := range encodings { + varNames := strings.Split(e.varName, ",") + all = append(all, varNames...) + varName := varNames[0] + switch { + case strings.HasPrefix(e.mapping, "http://encoding.spec.whatwg.org/"): + e.mapping = getWHATWG(e.mapping) + case strings.HasPrefix(e.mapping, "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/"): + e.mapping = getUCM(e.mapping) + } + + asciiSuperset, low := strings.HasPrefix(e.mapping, ascii), 0x00 + if asciiSuperset { + low = 0x80 + } + lvn := 1 + if strings.HasPrefix(varName, "ISO") || strings.HasPrefix(varName, "KOI") { + lvn = 3 + } + lowerVarName := strings.ToLower(varName[:lvn]) + varName[lvn:] + printf("// %s is the %s encoding.\n", varName, e.name) + if e.comment != "" { + printf("//\n// %s\n", e.comment) + } + printf("var %s *Charmap = &%s\n\nvar %s = Charmap{\nname: %q,\n", + varName, lowerVarName, lowerVarName, e.name) + if mibs[e.mib] { + log.Fatalf("MIB type %q declared multiple times.", e.mib) + } + printf("mib: identifier.%s,\n", e.mib) + printf("asciiSuperset: %t,\n", asciiSuperset) + printf("low: 0x%02x,\n", low) + printf("replacement: 0x%02x,\n", e.replacement) + + printf("decode: [256]utf8Enc{\n") + i, backMapping := 0, map[rune]byte{} + for _, c := range e.mapping { + if _, ok := backMapping[c]; !ok && c != utf8.RuneError { + backMapping[c] = byte(i) + } + var buf [8]byte + n := utf8.EncodeRune(buf[:], c) + if n > 3 { + panic(fmt.Sprintf("rune %q (%U) is too long", c, c)) + } + printf("{%d,[3]byte{0x%02x,0x%02x,0x%02x}},", n, buf[0], buf[1], buf[2]) + if i%2 == 1 { + printf("\n") + } + i++ + } + printf("},\n") + + printf("encode: [256]uint32{\n") + encode := make([]uint32, 0, 256) + for c, i := range backMapping { + encode = append(encode, uint32(i)<<24|uint32(c)) + } + sort.Sort(byRune(encode)) + for len(encode) < cap(encode) { + encode = append(encode, encode[len(encode)-1]) + } + for i, enc := range encode { + printf("0x%08x,", enc) + if i%8 == 7 { + printf("\n") + } + } + printf("},\n}\n") + + // Add an estimate of the size of a single Charmap{} struct value, which + // includes two 256 elem arrays of 4 bytes and some extra fields, which + // align to 3 uint64s on 64-bit architectures. + w.Size += 2*4*256 + 3*8 + } + // TODO: add proper line breaking. + printf("var listAll = []encoding.Encoding{\n%s,\n}\n\n", strings.Join(all, ",\n")) +} + +type byRune []uint32 + +func (b byRune) Len() int { return len(b) } +func (b byRune) Less(i, j int) bool { return b[i]&0xffffff < b[j]&0xffffff } +func (b byRune) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/htmlindex/gen.go b/vendor/golang.org/x/text/encoding/htmlindex/gen.go new file mode 100644 index 0000000000..ac6b4a77fd --- /dev/null +++ b/vendor/golang.org/x/text/encoding/htmlindex/gen.go @@ -0,0 +1,173 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "strings" + + "golang.org/x/text/internal/gen" +) + +type group struct { + Encodings []struct { + Labels []string + Name string + } +} + +func main() { + gen.Init() + + r := gen.Open("https://encoding.spec.whatwg.org", "whatwg", "encodings.json") + var groups []group + if err := json.NewDecoder(r).Decode(&groups); err != nil { + log.Fatalf("Error reading encodings.json: %v", err) + } + + w := &bytes.Buffer{} + fmt.Fprintln(w, "type htmlEncoding byte") + fmt.Fprintln(w, "const (") + for i, g := range groups { + for _, e := range g.Encodings { + key := strings.ToLower(e.Name) + name := consts[key] + if name == "" { + log.Fatalf("No const defined for %s.", key) + } + if i == 0 { + fmt.Fprintf(w, "%s htmlEncoding = iota\n", name) + } else { + fmt.Fprintf(w, "%s\n", name) + } + } + } + fmt.Fprintln(w, "numEncodings") + fmt.Fprint(w, ")\n\n") + + fmt.Fprintln(w, "var canonical = [numEncodings]string{") + for _, g := range groups { + for _, e := range g.Encodings { + fmt.Fprintf(w, "%q,\n", strings.ToLower(e.Name)) + } + } + fmt.Fprint(w, "}\n\n") + + fmt.Fprintln(w, "var nameMap = map[string]htmlEncoding{") + for _, g := range groups { + for _, e := range g.Encodings { + for _, l := range e.Labels { + key := strings.ToLower(e.Name) + name := consts[key] + fmt.Fprintf(w, "%q: %s,\n", l, name) + } + } + } + fmt.Fprint(w, "}\n\n") + + var tags []string + fmt.Fprintln(w, "var localeMap = []htmlEncoding{") + for _, loc := range locales { + tags = append(tags, loc.tag) + fmt.Fprintf(w, "%s, // %s \n", consts[loc.name], loc.tag) + } + fmt.Fprint(w, "}\n\n") + + fmt.Fprintf(w, "const locales = %q\n", strings.Join(tags, " ")) + + gen.WriteGoFile("tables.go", "htmlindex", w.Bytes()) +} + +// consts maps canonical encoding name to internal constant. +var consts = map[string]string{ + "utf-8": "utf8", + "ibm866": "ibm866", + "iso-8859-2": "iso8859_2", + "iso-8859-3": "iso8859_3", + "iso-8859-4": "iso8859_4", + "iso-8859-5": "iso8859_5", + "iso-8859-6": "iso8859_6", + "iso-8859-7": "iso8859_7", + "iso-8859-8": "iso8859_8", + "iso-8859-8-i": "iso8859_8I", + "iso-8859-10": "iso8859_10", + "iso-8859-13": "iso8859_13", + "iso-8859-14": "iso8859_14", + "iso-8859-15": "iso8859_15", + "iso-8859-16": "iso8859_16", + "koi8-r": "koi8r", + "koi8-u": "koi8u", + "macintosh": "macintosh", + "windows-874": "windows874", + "windows-1250": "windows1250", + "windows-1251": "windows1251", + "windows-1252": "windows1252", + "windows-1253": "windows1253", + "windows-1254": "windows1254", + "windows-1255": "windows1255", + "windows-1256": "windows1256", + "windows-1257": "windows1257", + "windows-1258": "windows1258", + "x-mac-cyrillic": "macintoshCyrillic", + "gbk": "gbk", + "gb18030": "gb18030", + // "hz-gb-2312": "hzgb2312", // Was removed from WhatWG + "big5": "big5", + "euc-jp": "eucjp", + "iso-2022-jp": "iso2022jp", + "shift_jis": "shiftJIS", + "euc-kr": "euckr", + "replacement": "replacement", + "utf-16be": "utf16be", + "utf-16le": "utf16le", + "x-user-defined": "xUserDefined", +} + +// locales is taken from +// https://html.spec.whatwg.org/multipage/syntax.html#encoding-sniffing-algorithm. +var locales = []struct{ tag, name string }{ + // The default value. Explicitly state latin to benefit from the exact + // script option, while still making 1252 the default encoding for languages + // written in Latin script. + {"und_Latn", "windows-1252"}, + {"ar", "windows-1256"}, + {"ba", "windows-1251"}, + {"be", "windows-1251"}, + {"bg", "windows-1251"}, + {"cs", "windows-1250"}, + {"el", "iso-8859-7"}, + {"et", "windows-1257"}, + {"fa", "windows-1256"}, + {"he", "windows-1255"}, + {"hr", "windows-1250"}, + {"hu", "iso-8859-2"}, + {"ja", "shift_jis"}, + {"kk", "windows-1251"}, + {"ko", "euc-kr"}, + {"ku", "windows-1254"}, + {"ky", "windows-1251"}, + {"lt", "windows-1257"}, + {"lv", "windows-1257"}, + {"mk", "windows-1251"}, + {"pl", "iso-8859-2"}, + {"ru", "windows-1251"}, + {"sah", "windows-1251"}, + {"sk", "windows-1250"}, + {"sl", "iso-8859-2"}, + {"sr", "windows-1251"}, + {"tg", "windows-1251"}, + {"th", "windows-874"}, + {"tr", "windows-1254"}, + {"tt", "windows-1251"}, + {"uk", "windows-1251"}, + {"vi", "windows-1258"}, + {"zh-hans", "gb18030"}, + {"zh-hant", "big5"}, +} diff --git a/vendor/golang.org/x/text/encoding/internal/identifier/gen.go b/vendor/golang.org/x/text/encoding/internal/identifier/gen.go new file mode 100644 index 0000000000..26cfef9c6b --- /dev/null +++ b/vendor/golang.org/x/text/encoding/internal/identifier/gen.go @@ -0,0 +1,142 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "log" + "strings" + + "golang.org/x/text/internal/gen" +) + +type registry struct { + XMLName xml.Name `xml:"registry"` + Updated string `xml:"updated"` + Registry []struct { + ID string `xml:"id,attr"` + Record []struct { + Name string `xml:"name"` + Xref []struct { + Type string `xml:"type,attr"` + Data string `xml:"data,attr"` + } `xml:"xref"` + Desc struct { + Data string `xml:",innerxml"` + // Any []struct { + // Data string `xml:",chardata"` + // } `xml:",any"` + // Data string `xml:",chardata"` + } `xml:"description,"` + MIB string `xml:"value"` + Alias []string `xml:"alias"` + MIME string `xml:"preferred_alias"` + } `xml:"record"` + } `xml:"registry"` +} + +func main() { + r := gen.OpenIANAFile("assignments/character-sets/character-sets.xml") + reg := ®istry{} + if err := xml.NewDecoder(r).Decode(®); err != nil && err != io.EOF { + log.Fatalf("Error decoding charset registry: %v", err) + } + if len(reg.Registry) == 0 || reg.Registry[0].ID != "character-sets-1" { + log.Fatalf("Unexpected ID %s", reg.Registry[0].ID) + } + + w := &bytes.Buffer{} + fmt.Fprintf(w, "const (\n") + for _, rec := range reg.Registry[0].Record { + constName := "" + for _, a := range rec.Alias { + if strings.HasPrefix(a, "cs") && strings.IndexByte(a, '-') == -1 { + // Some of the constant definitions have comments in them. Strip those. + constName = strings.Title(strings.SplitN(a[2:], "\n", 2)[0]) + } + } + if constName == "" { + switch rec.MIB { + case "2085": + constName = "HZGB2312" // Not listed as alias for some reason. + default: + log.Fatalf("No cs alias defined for %s.", rec.MIB) + } + } + if rec.MIME != "" { + rec.MIME = fmt.Sprintf(" (MIME: %s)", rec.MIME) + } + fmt.Fprintf(w, "// %s is the MIB identifier with IANA name %s%s.\n//\n", constName, rec.Name, rec.MIME) + if len(rec.Desc.Data) > 0 { + fmt.Fprint(w, "// ") + d := xml.NewDecoder(strings.NewReader(rec.Desc.Data)) + inElem := true + attr := "" + for { + t, err := d.Token() + if err != nil { + if err != io.EOF { + log.Fatal(err) + } + break + } + switch x := t.(type) { + case xml.CharData: + attr = "" // Don't need attribute info. + a := bytes.Split([]byte(x), []byte("\n")) + for i, b := range a { + if b = bytes.TrimSpace(b); len(b) != 0 { + if !inElem && i > 0 { + fmt.Fprint(w, "\n// ") + } + inElem = false + fmt.Fprintf(w, "%s ", string(b)) + } + } + case xml.StartElement: + if x.Name.Local == "xref" { + inElem = true + use := false + for _, a := range x.Attr { + if a.Name.Local == "type" { + use = use || a.Value != "person" + } + if a.Name.Local == "data" && use { + // Patch up URLs to use https. From some links, the + // https version is different from the http one. + s := a.Value + s = strings.Replace(s, "http://", "https://", -1) + s = strings.Replace(s, "/unicode/", "/", -1) + attr = s + " " + } + } + } + case xml.EndElement: + inElem = false + fmt.Fprint(w, attr) + } + } + fmt.Fprint(w, "\n") + } + for _, x := range rec.Xref { + switch x.Type { + case "rfc": + fmt.Fprintf(w, "// Reference: %s\n", strings.ToUpper(x.Data)) + case "uri": + fmt.Fprintf(w, "// Reference: %s\n", x.Data) + } + } + fmt.Fprintf(w, "%s MIB = %s\n", constName, rec.MIB) + fmt.Fprintln(w) + } + fmt.Fprintln(w, ")") + + gen.WriteGoFile("mib.go", "identifier", w.Bytes()) +} diff --git a/vendor/golang.org/x/text/encoding/japanese/maketables.go b/vendor/golang.org/x/text/encoding/japanese/maketables.go new file mode 100644 index 0000000000..023957a672 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/japanese/maketables.go @@ -0,0 +1,161 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates tables.go: +// go run maketables.go | gofmt > tables.go + +// TODO: Emoji extensions? +// https://www.unicode.org/faq/emoji_dingbats.html +// https://www.unicode.org/Public/UNIDATA/EmojiSources.txt + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" +) + +type entry struct { + jisCode, table int +} + +func main() { + fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") + fmt.Printf("// Package japanese provides Japanese encodings such as EUC-JP and Shift JIS.\n") + fmt.Printf(`package japanese // import "golang.org/x/text/encoding/japanese"` + "\n\n") + + reverse := [65536]entry{} + for i := range reverse { + reverse[i].table = -1 + } + + tables := []struct { + url string + name string + }{ + {"http://encoding.spec.whatwg.org/index-jis0208.txt", "0208"}, + {"http://encoding.spec.whatwg.org/index-jis0212.txt", "0212"}, + } + for i, table := range tables { + res, err := http.Get(table.url) + if err != nil { + log.Fatalf("%q: Get: %v", table.url, err) + } + defer res.Body.Close() + + mapping := [65536]uint16{} + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := 0, uint16(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("%q: could not parse %q", table.url, s) + } + if x < 0 || 120*94 <= x { + log.Fatalf("%q: JIS code %d is out of range", table.url, x) + } + mapping[x] = y + if reverse[y].table == -1 { + reverse[y] = entry{jisCode: x, table: i} + } + } + if err := scanner.Err(); err != nil { + log.Fatalf("%q: scanner error: %v", table.url, err) + } + + fmt.Printf("// jis%sDecode is the decoding table from JIS %s code to Unicode.\n// It is defined at %s\n", + table.name, table.name, table.url) + fmt.Printf("var jis%sDecode = [...]uint16{\n", table.name) + for i, m := range mapping { + if m != 0 { + fmt.Printf("\t%d: 0x%04X,\n", i, m) + } + } + fmt.Printf("}\n\n") + } + + // Any run of at least separation continuous zero entries in the reverse map will + // be a separate encode table. + const separation = 1024 + + intervals := []interval(nil) + low, high := -1, -1 + for i, v := range reverse { + if v.table == -1 { + continue + } + if low < 0 { + low = i + } else if i-high >= separation { + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + low = i + } + high = i + 1 + } + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + sort.Sort(byDecreasingLength(intervals)) + + fmt.Printf("const (\n") + fmt.Printf("\tjis0208 = 1\n") + fmt.Printf("\tjis0212 = 2\n") + fmt.Printf("\tcodeMask = 0x7f\n") + fmt.Printf("\tcodeShift = 7\n") + fmt.Printf("\ttableShift = 14\n") + fmt.Printf(")\n\n") + + fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) + fmt.Printf("// encodeX are the encoding tables from Unicode to JIS code,\n") + fmt.Printf("// sorted by decreasing length.\n") + for i, v := range intervals { + fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) + } + fmt.Printf("//\n") + fmt.Printf("// The high two bits of the value record whether the JIS code comes from the\n") + fmt.Printf("// JIS0208 table (high bits == 1) or the JIS0212 table (high bits == 2).\n") + fmt.Printf("// The low 14 bits are two 7-bit unsigned integers j1 and j2 that form the\n") + fmt.Printf("// JIS code (94*j1 + j2) within that table.\n") + fmt.Printf("\n") + + for i, v := range intervals { + fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) + fmt.Printf("var encode%d = [...]uint16{\n", i) + for j := v.low; j < v.high; j++ { + x := reverse[j] + if x.table == -1 { + continue + } + fmt.Printf("\t%d - %d: jis%s<<14 | 0x%02X<<7 | 0x%02X,\n", + j, v.low, tables[x.table].name, x.jisCode/94, x.jisCode%94) + } + fmt.Printf("}\n\n") + } +} + +// interval is a half-open interval [low, high). +type interval struct { + low, high int +} + +func (i interval) len() int { return i.high - i.low } + +// byDecreasingLength sorts intervals by decreasing length. +type byDecreasingLength []interval + +func (b byDecreasingLength) Len() int { return len(b) } +func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } +func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/korean/maketables.go b/vendor/golang.org/x/text/encoding/korean/maketables.go new file mode 100644 index 0000000000..c84034fb67 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/korean/maketables.go @@ -0,0 +1,143 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates tables.go: +// go run maketables.go | gofmt > tables.go + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" +) + +func main() { + fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") + fmt.Printf("// Package korean provides Korean encodings such as EUC-KR.\n") + fmt.Printf(`package korean // import "golang.org/x/text/encoding/korean"` + "\n\n") + + res, err := http.Get("http://encoding.spec.whatwg.org/index-euc-kr.txt") + if err != nil { + log.Fatalf("Get: %v", err) + } + defer res.Body.Close() + + mapping := [65536]uint16{} + reverse := [65536]uint16{} + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := uint16(0), uint16(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0 || 178*(0xc7-0x81)+(0xfe-0xc7)*94+(0xff-0xa1) <= x { + log.Fatalf("EUC-KR code %d is out of range", x) + } + mapping[x] = y + if reverse[y] == 0 { + c0, c1 := uint16(0), uint16(0) + if x < 178*(0xc7-0x81) { + c0 = uint16(x/178) + 0x81 + c1 = uint16(x % 178) + switch { + case c1 < 1*26: + c1 += 0x41 + case c1 < 2*26: + c1 += 0x47 + default: + c1 += 0x4d + } + } else { + x -= 178 * (0xc7 - 0x81) + c0 = uint16(x/94) + 0xc7 + c1 = uint16(x%94) + 0xa1 + } + reverse[y] = c0<<8 | c1 + } + } + if err := scanner.Err(); err != nil { + log.Fatalf("scanner error: %v", err) + } + + fmt.Printf("// decode is the decoding table from EUC-KR code to Unicode.\n") + fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-euc-kr.txt\n") + fmt.Printf("var decode = [...]uint16{\n") + for i, v := range mapping { + if v != 0 { + fmt.Printf("\t%d: 0x%04X,\n", i, v) + } + } + fmt.Printf("}\n\n") + + // Any run of at least separation continuous zero entries in the reverse map will + // be a separate encode table. + const separation = 1024 + + intervals := []interval(nil) + low, high := -1, -1 + for i, v := range reverse { + if v == 0 { + continue + } + if low < 0 { + low = i + } else if i-high >= separation { + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + low = i + } + high = i + 1 + } + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + sort.Sort(byDecreasingLength(intervals)) + + fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) + fmt.Printf("// encodeX are the encoding tables from Unicode to EUC-KR code,\n") + fmt.Printf("// sorted by decreasing length.\n") + for i, v := range intervals { + fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) + } + fmt.Printf("\n") + + for i, v := range intervals { + fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) + fmt.Printf("var encode%d = [...]uint16{\n", i) + for j := v.low; j < v.high; j++ { + x := reverse[j] + if x == 0 { + continue + } + fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) + } + fmt.Printf("}\n\n") + } +} + +// interval is a half-open interval [low, high). +type interval struct { + low, high int +} + +func (i interval) len() int { return i.high - i.low } + +// byDecreasingLength sorts intervals by decreasing length. +type byDecreasingLength []interval + +func (b byDecreasingLength) Len() int { return len(b) } +func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } +func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go b/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go new file mode 100644 index 0000000000..55016c7862 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go @@ -0,0 +1,161 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates tables.go: +// go run maketables.go | gofmt > tables.go + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" +) + +func main() { + fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") + fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n") + fmt.Printf(`package simplifiedchinese // import "golang.org/x/text/encoding/simplifiedchinese"` + "\n\n") + + printGB18030() + printGBK() +} + +func printGB18030() { + res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt") + if err != nil { + log.Fatalf("Get: %v", err) + } + defer res.Body.Close() + + fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n") + fmt.Printf("var gb18030 = [...][2]uint16{\n") + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := uint32(0), uint32(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0x10000 && y < 0x10000 { + fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y) + } + } + fmt.Printf("}\n\n") +} + +func printGBK() { + res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt") + if err != nil { + log.Fatalf("Get: %v", err) + } + defer res.Body.Close() + + mapping := [65536]uint16{} + reverse := [65536]uint16{} + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := uint16(0), uint16(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0 || 126*190 <= x { + log.Fatalf("GBK code %d is out of range", x) + } + mapping[x] = y + if reverse[y] == 0 { + c0, c1 := x/190, x%190 + if c1 >= 0x3f { + c1++ + } + reverse[y] = (0x81+c0)<<8 | (0x40 + c1) + } + } + if err := scanner.Err(); err != nil { + log.Fatalf("scanner error: %v", err) + } + + fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n") + fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n") + fmt.Printf("var decode = [...]uint16{\n") + for i, v := range mapping { + if v != 0 { + fmt.Printf("\t%d: 0x%04X,\n", i, v) + } + } + fmt.Printf("}\n\n") + + // Any run of at least separation continuous zero entries in the reverse map will + // be a separate encode table. + const separation = 1024 + + intervals := []interval(nil) + low, high := -1, -1 + for i, v := range reverse { + if v == 0 { + continue + } + if low < 0 { + low = i + } else if i-high >= separation { + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + low = i + } + high = i + 1 + } + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + sort.Sort(byDecreasingLength(intervals)) + + fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) + fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n") + fmt.Printf("// sorted by decreasing length.\n") + for i, v := range intervals { + fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) + } + fmt.Printf("\n") + + for i, v := range intervals { + fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) + fmt.Printf("var encode%d = [...]uint16{\n", i) + for j := v.low; j < v.high; j++ { + x := reverse[j] + if x == 0 { + continue + } + fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) + } + fmt.Printf("}\n\n") + } +} + +// interval is a half-open interval [low, high). +type interval struct { + low, high int +} + +func (i interval) len() int { return i.high - i.low } + +// byDecreasingLength sorts intervals by decreasing length. +type byDecreasingLength []interval + +func (b byDecreasingLength) Len() int { return len(b) } +func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } +func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go b/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go new file mode 100644 index 0000000000..cf7fdb31a5 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go @@ -0,0 +1,140 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates tables.go: +// go run maketables.go | gofmt > tables.go + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" +) + +func main() { + fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") + fmt.Printf("// Package traditionalchinese provides Traditional Chinese encodings such as Big5.\n") + fmt.Printf(`package traditionalchinese // import "golang.org/x/text/encoding/traditionalchinese"` + "\n\n") + + res, err := http.Get("http://encoding.spec.whatwg.org/index-big5.txt") + if err != nil { + log.Fatalf("Get: %v", err) + } + defer res.Body.Close() + + mapping := [65536]uint32{} + reverse := [65536 * 4]uint16{} + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := uint16(0), uint32(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0 || 126*157 <= x { + log.Fatalf("Big5 code %d is out of range", x) + } + mapping[x] = y + + // The WHATWG spec http://encoding.spec.whatwg.org/#indexes says that + // "The index pointer for code point in index is the first pointer + // corresponding to code point in index", which would normally mean + // that the code below should be guarded by "if reverse[y] == 0", but + // last instead of first seems to match the behavior of + // "iconv -f UTF-8 -t BIG5". For example, U+8005 者 occurs twice in + // http://encoding.spec.whatwg.org/index-big5.txt, as index 2148 + // (encoded as "\x8e\xcd") and index 6543 (encoded as "\xaa\xcc") + // and "echo 者 | iconv -f UTF-8 -t BIG5 | xxd" gives "\xaa\xcc". + c0, c1 := x/157, x%157 + if c1 < 0x3f { + c1 += 0x40 + } else { + c1 += 0x62 + } + reverse[y] = (0x81+c0)<<8 | c1 + } + if err := scanner.Err(); err != nil { + log.Fatalf("scanner error: %v", err) + } + + fmt.Printf("// decode is the decoding table from Big5 code to Unicode.\n") + fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-big5.txt\n") + fmt.Printf("var decode = [...]uint32{\n") + for i, v := range mapping { + if v != 0 { + fmt.Printf("\t%d: 0x%08X,\n", i, v) + } + } + fmt.Printf("}\n\n") + + // Any run of at least separation continuous zero entries in the reverse map will + // be a separate encode table. + const separation = 1024 + + intervals := []interval(nil) + low, high := -1, -1 + for i, v := range reverse { + if v == 0 { + continue + } + if low < 0 { + low = i + } else if i-high >= separation { + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + low = i + } + high = i + 1 + } + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + sort.Sort(byDecreasingLength(intervals)) + + fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) + fmt.Printf("// encodeX are the encoding tables from Unicode to Big5 code,\n") + fmt.Printf("// sorted by decreasing length.\n") + for i, v := range intervals { + fmt.Printf("// encode%d: %5d entries for runes in [%6d, %6d).\n", i, v.len(), v.low, v.high) + } + fmt.Printf("\n") + + for i, v := range intervals { + fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) + fmt.Printf("var encode%d = [...]uint16{\n", i) + for j := v.low; j < v.high; j++ { + x := reverse[j] + if x == 0 { + continue + } + fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) + } + fmt.Printf("}\n\n") + } +} + +// interval is a half-open interval [low, high). +type interval struct { + low, high int +} + +func (i interval) len() int { return i.high - i.low } + +// byDecreasingLength sorts intervals by decreasing length. +type byDecreasingLength []interval + +func (b byDecreasingLength) Len() int { return len(b) } +func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } +func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/internal/language/compact/gen.go b/vendor/golang.org/x/text/internal/language/compact/gen.go new file mode 100644 index 0000000000..0c36a052f6 --- /dev/null +++ b/vendor/golang.org/x/text/internal/language/compact/gen.go @@ -0,0 +1,64 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Language tag table generator. +// Data read from the web. + +package main + +import ( + "flag" + "fmt" + "log" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/unicode/cldr" +) + +var ( + test = flag.Bool("test", + false, + "test existing tables; can be used to compare web data with package data.") + outputFile = flag.String("output", + "tables.go", + "output file for generated tables") +) + +func main() { + gen.Init() + + w := gen.NewCodeWriter() + defer w.WriteGoFile("tables.go", "compact") + + fmt.Fprintln(w, `import "golang.org/x/text/internal/language"`) + + b := newBuilder(w) + gen.WriteCLDRVersion(w) + + b.writeCompactIndex() +} + +type builder struct { + w *gen.CodeWriter + data *cldr.CLDR + supp *cldr.SupplementalData +} + +func newBuilder(w *gen.CodeWriter) *builder { + r := gen.OpenCLDRCoreZip() + defer r.Close() + d := &cldr.Decoder{} + data, err := d.DecodeZip(r) + if err != nil { + log.Fatal(err) + } + b := builder{ + w: w, + data: data, + supp: data.Supplemental(), + } + return &b +} diff --git a/vendor/golang.org/x/text/internal/language/compact/gen_index.go b/vendor/golang.org/x/text/internal/language/compact/gen_index.go new file mode 100644 index 0000000000..136cefaf08 --- /dev/null +++ b/vendor/golang.org/x/text/internal/language/compact/gen_index.go @@ -0,0 +1,113 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This file generates derivative tables based on the language package itself. + +import ( + "fmt" + "log" + "sort" + "strings" + + "golang.org/x/text/internal/language" +) + +// Compact indices: +// Note -va-X variants only apply to localization variants. +// BCP variants only ever apply to language. +// The only ambiguity between tags is with regions. + +func (b *builder) writeCompactIndex() { + // Collect all language tags for which we have any data in CLDR. + m := map[language.Tag]bool{} + for _, lang := range b.data.Locales() { + // We include all locales unconditionally to be consistent with en_US. + // We want en_US, even though it has no data associated with it. + + // TODO: put any of the languages for which no data exists at the end + // of the index. This allows all components based on ICU to use that + // as the cutoff point. + // if x := data.RawLDML(lang); false || + // x.LocaleDisplayNames != nil || + // x.Characters != nil || + // x.Delimiters != nil || + // x.Measurement != nil || + // x.Dates != nil || + // x.Numbers != nil || + // x.Units != nil || + // x.ListPatterns != nil || + // x.Collations != nil || + // x.Segmentations != nil || + // x.Rbnf != nil || + // x.Annotations != nil || + // x.Metadata != nil { + + // TODO: support POSIX natively, albeit non-standard. + tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1)) + m[tag] = true + // } + } + + // TODO: plural rules are also defined for the deprecated tags: + // iw mo sh tl + // Consider removing these as compact tags. + + // Include locales for plural rules, which uses a different structure. + for _, plurals := range b.supp.Plurals { + for _, rules := range plurals.PluralRules { + for _, lang := range strings.Split(rules.Locales, " ") { + m[language.Make(lang)] = true + } + } + } + + var coreTags []language.CompactCoreInfo + var special []string + + for t := range m { + if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" { + log.Fatalf("Unexpected extension %v in %v", x, t) + } + if len(t.Variants()) == 0 && len(t.Extensions()) == 0 { + cci, ok := language.GetCompactCore(t) + if !ok { + log.Fatalf("Locale for non-basic language %q", t) + } + coreTags = append(coreTags, cci) + } else { + special = append(special, t.String()) + } + } + + w := b.w + + sort.Slice(coreTags, func(i, j int) bool { return coreTags[i] < coreTags[j] }) + sort.Strings(special) + + w.WriteComment(` + NumCompactTags is the number of common tags. The maximum tag is + NumCompactTags-1.`) + w.WriteConst("NumCompactTags", len(m)) + + fmt.Fprintln(w, "const (") + for i, t := range coreTags { + fmt.Fprintf(w, "%s ID = %d\n", ident(t.Tag().String()), i) + } + for i, t := range special { + fmt.Fprintf(w, "%s ID = %d\n", ident(t), i+len(coreTags)) + } + fmt.Fprintln(w, ")") + + w.WriteVar("coreTags", coreTags) + + w.WriteConst("specialTagsStr", strings.Join(special, " ")) +} + +func ident(s string) string { + return strings.Replace(s, "-", "", -1) + "Index" +} diff --git a/vendor/golang.org/x/text/internal/language/compact/gen_parents.go b/vendor/golang.org/x/text/internal/language/compact/gen_parents.go new file mode 100644 index 0000000000..9543d58323 --- /dev/null +++ b/vendor/golang.org/x/text/internal/language/compact/gen_parents.go @@ -0,0 +1,54 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "log" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/language" + "golang.org/x/text/internal/language/compact" + "golang.org/x/text/unicode/cldr" +) + +func main() { + r := gen.OpenCLDRCoreZip() + defer r.Close() + + d := &cldr.Decoder{} + data, err := d.DecodeZip(r) + if err != nil { + log.Fatalf("DecodeZip: %v", err) + } + + w := gen.NewCodeWriter() + defer w.WriteGoFile("parents.go", "compact") + + // Create parents table. + type ID uint16 + parents := make([]ID, compact.NumCompactTags) + for _, loc := range data.Locales() { + tag := language.MustParse(loc) + index, ok := compact.FromTag(tag) + if !ok { + continue + } + parentIndex := compact.ID(0) // und + for p := tag.Parent(); p != language.Und; p = p.Parent() { + if x, ok := compact.FromTag(p); ok { + parentIndex = x + break + } + } + parents[index] = ID(parentIndex) + } + + w.WriteComment(` + parents maps a compact index of a tag to the compact index of the parent of + this tag.`) + w.WriteVar("parents", parents) +} diff --git a/vendor/golang.org/x/text/internal/language/gen.go b/vendor/golang.org/x/text/internal/language/gen.go new file mode 100644 index 0000000000..cdcc7febcb --- /dev/null +++ b/vendor/golang.org/x/text/internal/language/gen.go @@ -0,0 +1,1520 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Language tag table generator. +// Data read from the web. + +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/tag" + "golang.org/x/text/unicode/cldr" +) + +var ( + test = flag.Bool("test", + false, + "test existing tables; can be used to compare web data with package data.") + outputFile = flag.String("output", + "tables.go", + "output file for generated tables") +) + +var comment = []string{ + ` +lang holds an alphabetically sorted list of ISO-639 language identifiers. +All entries are 4 bytes. The index of the identifier (divided by 4) is the language tag. +For 2-byte language identifiers, the two successive bytes have the following meaning: + - if the first letter of the 2- and 3-letter ISO codes are the same: + the second and third letter of the 3-letter ISO code. + - otherwise: a 0 and a by 2 bits right-shifted index into altLangISO3. +For 3-byte language identifiers the 4th byte is 0.`, + ` +langNoIndex is a bit vector of all 3-letter language codes that are not used as an index +in lookup tables. The language ids for these language codes are derived directly +from the letters and are not consecutive.`, + ` +altLangISO3 holds an alphabetically sorted list of 3-letter language code alternatives +to 2-letter language codes that cannot be derived using the method described above. +Each 3-letter code is followed by its 1-byte langID.`, + ` +altLangIndex is used to convert indexes in altLangISO3 to langIDs.`, + ` +AliasMap maps langIDs to their suggested replacements.`, + ` +script is an alphabetically sorted list of ISO 15924 codes. The index +of the script in the string, divided by 4, is the internal scriptID.`, + ` +isoRegionOffset needs to be added to the index of regionISO to obtain the regionID +for 2-letter ISO codes. (The first isoRegionOffset regionIDs are reserved for +the UN.M49 codes used for groups.)`, + ` +regionISO holds a list of alphabetically sorted 2-letter ISO region codes. +Each 2-letter codes is followed by two bytes with the following meaning: + - [A-Z}{2}: the first letter of the 2-letter code plus these two + letters form the 3-letter ISO code. + - 0, n: index into altRegionISO3.`, + ` +regionTypes defines the status of a region for various standards.`, + ` +m49 maps regionIDs to UN.M49 codes. The first isoRegionOffset entries are +codes indicating collections of regions.`, + ` +m49Index gives indexes into fromM49 based on the three most significant bits +of a 10-bit UN.M49 code. To search an UN.M49 code in fromM49, search in + fromM49[m49Index[msb39(code)]:m49Index[msb3(code)+1]] +for an entry where the first 7 bits match the 7 lsb of the UN.M49 code. +The region code is stored in the 9 lsb of the indexed value.`, + ` +fromM49 contains entries to map UN.M49 codes to regions. See m49Index for details.`, + ` +altRegionISO3 holds a list of 3-letter region codes that cannot be +mapped to 2-letter codes using the default algorithm. This is a short list.`, + ` +altRegionIDs holds a list of regionIDs the positions of which match those +of the 3-letter ISO codes in altRegionISO3.`, + ` +variantNumSpecialized is the number of specialized variants in variants.`, + ` +suppressScript is an index from langID to the dominant script for that language, +if it exists. If a script is given, it should be suppressed from the language tag.`, + ` +likelyLang is a lookup table, indexed by langID, for the most likely +scripts and regions given incomplete information. If more entries exist for a +given language, region and script are the index and size respectively +of the list in likelyLangList.`, + ` +likelyLangList holds lists info associated with likelyLang.`, + ` +likelyRegion is a lookup table, indexed by regionID, for the most likely +languages and scripts given incomplete information. If more entries exist +for a given regionID, lang and script are the index and size respectively +of the list in likelyRegionList. +TODO: exclude containers and user-definable regions from the list.`, + ` +likelyRegionList holds lists info associated with likelyRegion.`, + ` +likelyScript is a lookup table, indexed by scriptID, for the most likely +languages and regions given a script.`, + ` +nRegionGroups is the number of region groups.`, + ` +regionInclusion maps region identifiers to sets of regions in regionInclusionBits, +where each set holds all groupings that are directly connected in a region +containment graph.`, + ` +regionInclusionBits is an array of bit vectors where every vector represents +a set of region groupings. These sets are used to compute the distance +between two regions for the purpose of language matching.`, + ` +regionInclusionNext marks, for each entry in regionInclusionBits, the set of +all groups that are reachable from the groups set in the respective entry.`, +} + +// TODO: consider changing some of these structures to tries. This can reduce +// memory, but may increase the need for memory allocations. This could be +// mitigated if we can piggyback on language tags for common cases. + +func failOnError(e error) { + if e != nil { + log.Panic(e) + } +} + +type setType int + +const ( + Indexed setType = 1 + iota // all elements must be of same size + Linear +) + +type stringSet struct { + s []string + sorted, frozen bool + + // We often need to update values after the creation of an index is completed. + // We include a convenience map for keeping track of this. + update map[string]string + typ setType // used for checking. +} + +func (ss *stringSet) clone() stringSet { + c := *ss + c.s = append([]string(nil), c.s...) + return c +} + +func (ss *stringSet) setType(t setType) { + if ss.typ != t && ss.typ != 0 { + log.Panicf("type %d cannot be assigned as it was already %d", t, ss.typ) + } +} + +// parse parses a whitespace-separated string and initializes ss with its +// components. +func (ss *stringSet) parse(s string) { + scan := bufio.NewScanner(strings.NewReader(s)) + scan.Split(bufio.ScanWords) + for scan.Scan() { + ss.add(scan.Text()) + } +} + +func (ss *stringSet) assertChangeable() { + if ss.frozen { + log.Panic("attempt to modify a frozen stringSet") + } +} + +func (ss *stringSet) add(s string) { + ss.assertChangeable() + ss.s = append(ss.s, s) + ss.sorted = ss.frozen +} + +func (ss *stringSet) freeze() { + ss.compact() + ss.frozen = true +} + +func (ss *stringSet) compact() { + if ss.sorted { + return + } + a := ss.s + sort.Strings(a) + k := 0 + for i := 1; i < len(a); i++ { + if a[k] != a[i] { + a[k+1] = a[i] + k++ + } + } + ss.s = a[:k+1] + ss.sorted = ss.frozen +} + +type funcSorter struct { + fn func(a, b string) bool + sort.StringSlice +} + +func (s funcSorter) Less(i, j int) bool { + return s.fn(s.StringSlice[i], s.StringSlice[j]) +} + +func (ss *stringSet) sortFunc(f func(a, b string) bool) { + ss.compact() + sort.Sort(funcSorter{f, sort.StringSlice(ss.s)}) +} + +func (ss *stringSet) remove(s string) { + ss.assertChangeable() + if i, ok := ss.find(s); ok { + copy(ss.s[i:], ss.s[i+1:]) + ss.s = ss.s[:len(ss.s)-1] + } +} + +func (ss *stringSet) replace(ol, nu string) { + ss.s[ss.index(ol)] = nu + ss.sorted = ss.frozen +} + +func (ss *stringSet) index(s string) int { + ss.setType(Indexed) + i, ok := ss.find(s) + if !ok { + if i < len(ss.s) { + log.Panicf("find: item %q is not in list. Closest match is %q.", s, ss.s[i]) + } + log.Panicf("find: item %q is not in list", s) + + } + return i +} + +func (ss *stringSet) find(s string) (int, bool) { + ss.compact() + i := sort.SearchStrings(ss.s, s) + return i, i != len(ss.s) && ss.s[i] == s +} + +func (ss *stringSet) slice() []string { + ss.compact() + return ss.s +} + +func (ss *stringSet) updateLater(v, key string) { + if ss.update == nil { + ss.update = map[string]string{} + } + ss.update[v] = key +} + +// join joins the string and ensures that all entries are of the same length. +func (ss *stringSet) join() string { + ss.setType(Indexed) + n := len(ss.s[0]) + for _, s := range ss.s { + if len(s) != n { + log.Panicf("join: not all entries are of the same length: %q", s) + } + } + ss.s = append(ss.s, strings.Repeat("\xff", n)) + return strings.Join(ss.s, "") +} + +// ianaEntry holds information for an entry in the IANA Language Subtag Repository. +// All types use the same entry. +// See http://tools.ietf.org/html/bcp47#section-5.1 for a description of the various +// fields. +type ianaEntry struct { + typ string + description []string + scope string + added string + preferred string + deprecated string + suppressScript string + macro string + prefix []string +} + +type builder struct { + w *gen.CodeWriter + hw io.Writer // MultiWriter for w and w.Hash + data *cldr.CLDR + supp *cldr.SupplementalData + + // indices + locale stringSet // common locales + lang stringSet // canonical language ids (2 or 3 letter ISO codes) with data + langNoIndex stringSet // 3-letter ISO codes with no associated data + script stringSet // 4-letter ISO codes + region stringSet // 2-letter ISO or 3-digit UN M49 codes + variant stringSet // 4-8-alphanumeric variant code. + + // Region codes that are groups with their corresponding group IDs. + groups map[int]index + + // langInfo + registry map[string]*ianaEntry +} + +type index uint + +func newBuilder(w *gen.CodeWriter) *builder { + r := gen.OpenCLDRCoreZip() + defer r.Close() + d := &cldr.Decoder{} + data, err := d.DecodeZip(r) + failOnError(err) + b := builder{ + w: w, + hw: io.MultiWriter(w, w.Hash), + data: data, + supp: data.Supplemental(), + } + b.parseRegistry() + return &b +} + +func (b *builder) parseRegistry() { + r := gen.OpenIANAFile("assignments/language-subtag-registry") + defer r.Close() + b.registry = make(map[string]*ianaEntry) + + scan := bufio.NewScanner(r) + scan.Split(bufio.ScanWords) + var record *ianaEntry + for more := scan.Scan(); more; { + key := scan.Text() + more = scan.Scan() + value := scan.Text() + switch key { + case "Type:": + record = &ianaEntry{typ: value} + case "Subtag:", "Tag:": + if s := strings.SplitN(value, "..", 2); len(s) > 1 { + for a := s[0]; a <= s[1]; a = inc(a) { + b.addToRegistry(a, record) + } + } else { + b.addToRegistry(value, record) + } + case "Suppress-Script:": + record.suppressScript = value + case "Added:": + record.added = value + case "Deprecated:": + record.deprecated = value + case "Macrolanguage:": + record.macro = value + case "Preferred-Value:": + record.preferred = value + case "Prefix:": + record.prefix = append(record.prefix, value) + case "Scope:": + record.scope = value + case "Description:": + buf := []byte(value) + for more = scan.Scan(); more; more = scan.Scan() { + b := scan.Bytes() + if b[0] == '%' || b[len(b)-1] == ':' { + break + } + buf = append(buf, ' ') + buf = append(buf, b...) + } + record.description = append(record.description, string(buf)) + continue + default: + continue + } + more = scan.Scan() + } + if scan.Err() != nil { + log.Panic(scan.Err()) + } +} + +func (b *builder) addToRegistry(key string, entry *ianaEntry) { + if info, ok := b.registry[key]; ok { + if info.typ != "language" || entry.typ != "extlang" { + log.Fatalf("parseRegistry: tag %q already exists", key) + } + } else { + b.registry[key] = entry + } +} + +var commentIndex = make(map[string]string) + +func init() { + for _, s := range comment { + key := strings.TrimSpace(strings.SplitN(s, " ", 2)[0]) + commentIndex[key] = s + } +} + +func (b *builder) comment(name string) { + if s := commentIndex[name]; len(s) > 0 { + b.w.WriteComment(s) + } else { + fmt.Fprintln(b.w) + } +} + +func (b *builder) pf(f string, x ...interface{}) { + fmt.Fprintf(b.hw, f, x...) + fmt.Fprint(b.hw, "\n") +} + +func (b *builder) p(x ...interface{}) { + fmt.Fprintln(b.hw, x...) +} + +func (b *builder) addSize(s int) { + b.w.Size += s + b.pf("// Size: %d bytes", s) +} + +func (b *builder) writeConst(name string, x interface{}) { + b.comment(name) + b.w.WriteConst(name, x) +} + +// writeConsts computes f(v) for all v in values and writes the results +// as constants named _v to a single constant block. +func (b *builder) writeConsts(f func(string) int, values ...string) { + b.pf("const (") + for _, v := range values { + b.pf("\t_%s = %v", v, f(v)) + } + b.pf(")") +} + +// writeType writes the type of the given value, which must be a struct. +func (b *builder) writeType(value interface{}) { + b.comment(reflect.TypeOf(value).Name()) + b.w.WriteType(value) +} + +func (b *builder) writeSlice(name string, ss interface{}) { + b.writeSliceAddSize(name, 0, ss) +} + +func (b *builder) writeSliceAddSize(name string, extraSize int, ss interface{}) { + b.comment(name) + b.w.Size += extraSize + v := reflect.ValueOf(ss) + t := v.Type().Elem() + b.pf("// Size: %d bytes, %d elements", v.Len()*int(t.Size())+extraSize, v.Len()) + + fmt.Fprintf(b.w, "var %s = ", name) + b.w.WriteArray(ss) + b.p() +} + +type FromTo struct { + From, To uint16 +} + +func (b *builder) writeSortedMap(name string, ss *stringSet, index func(s string) uint16) { + ss.sortFunc(func(a, b string) bool { + return index(a) < index(b) + }) + m := []FromTo{} + for _, s := range ss.s { + m = append(m, FromTo{index(s), index(ss.update[s])}) + } + b.writeSlice(name, m) +} + +const base = 'z' - 'a' + 1 + +func strToInt(s string) uint { + v := uint(0) + for i := 0; i < len(s); i++ { + v *= base + v += uint(s[i] - 'a') + } + return v +} + +// converts the given integer to the original ASCII string passed to strToInt. +// len(s) must match the number of characters obtained. +func intToStr(v uint, s []byte) { + for i := len(s) - 1; i >= 0; i-- { + s[i] = byte(v%base) + 'a' + v /= base + } +} + +func (b *builder) writeBitVector(name string, ss []string) { + vec := make([]uint8, int(math.Ceil(math.Pow(base, float64(len(ss[0])))/8))) + for _, s := range ss { + v := strToInt(s) + vec[v/8] |= 1 << (v % 8) + } + b.writeSlice(name, vec) +} + +// TODO: convert this type into a list or two-stage trie. +func (b *builder) writeMapFunc(name string, m map[string]string, f func(string) uint16) { + b.comment(name) + v := reflect.ValueOf(m) + sz := v.Len() * (2 + int(v.Type().Key().Size())) + for _, k := range m { + sz += len(k) + } + b.addSize(sz) + keys := []string{} + b.pf(`var %s = map[string]uint16{`, name) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + b.pf("\t%q: %v,", k, f(m[k])) + } + b.p("}") +} + +func (b *builder) writeMap(name string, m interface{}) { + b.comment(name) + v := reflect.ValueOf(m) + sz := v.Len() * (2 + int(v.Type().Key().Size()) + int(v.Type().Elem().Size())) + b.addSize(sz) + f := strings.FieldsFunc(fmt.Sprintf("%#v", m), func(r rune) bool { + return strings.IndexRune("{}, ", r) != -1 + }) + sort.Strings(f[1:]) + b.pf(`var %s = %s{`, name, f[0]) + for _, kv := range f[1:] { + b.pf("\t%s,", kv) + } + b.p("}") +} + +func (b *builder) langIndex(s string) uint16 { + if s == "und" { + return 0 + } + if i, ok := b.lang.find(s); ok { + return uint16(i) + } + return uint16(strToInt(s)) + uint16(len(b.lang.s)) +} + +// inc advances the string to its lexicographical successor. +func inc(s string) string { + const maxTagLength = 4 + var buf [maxTagLength]byte + intToStr(strToInt(strings.ToLower(s))+1, buf[:len(s)]) + for i := 0; i < len(s); i++ { + if s[i] <= 'Z' { + buf[i] -= 'a' - 'A' + } + } + return string(buf[:len(s)]) +} + +func (b *builder) parseIndices() { + meta := b.supp.Metadata + + for k, v := range b.registry { + var ss *stringSet + switch v.typ { + case "language": + if len(k) == 2 || v.suppressScript != "" || v.scope == "special" { + b.lang.add(k) + continue + } else { + ss = &b.langNoIndex + } + case "region": + ss = &b.region + case "script": + ss = &b.script + case "variant": + ss = &b.variant + default: + continue + } + ss.add(k) + } + // Include any language for which there is data. + for _, lang := range b.data.Locales() { + if x := b.data.RawLDML(lang); false || + x.LocaleDisplayNames != nil || + x.Characters != nil || + x.Delimiters != nil || + x.Measurement != nil || + x.Dates != nil || + x.Numbers != nil || + x.Units != nil || + x.ListPatterns != nil || + x.Collations != nil || + x.Segmentations != nil || + x.Rbnf != nil || + x.Annotations != nil || + x.Metadata != nil { + + from := strings.Split(lang, "_") + if lang := from[0]; lang != "root" { + b.lang.add(lang) + } + } + } + // Include locales for plural rules, which uses a different structure. + for _, plurals := range b.data.Supplemental().Plurals { + for _, rules := range plurals.PluralRules { + for _, lang := range strings.Split(rules.Locales, " ") { + if lang = strings.Split(lang, "_")[0]; lang != "root" { + b.lang.add(lang) + } + } + } + } + // Include languages in likely subtags. + for _, m := range b.supp.LikelySubtags.LikelySubtag { + from := strings.Split(m.From, "_") + b.lang.add(from[0]) + } + // Include ISO-639 alpha-3 bibliographic entries. + for _, a := range meta.Alias.LanguageAlias { + if a.Reason == "bibliographic" { + b.langNoIndex.add(a.Type) + } + } + // Include regions in territoryAlias (not all are in the IANA registry!) + for _, reg := range b.supp.Metadata.Alias.TerritoryAlias { + if len(reg.Type) == 2 { + b.region.add(reg.Type) + } + } + + for _, s := range b.lang.s { + if len(s) == 3 { + b.langNoIndex.remove(s) + } + } + b.writeConst("NumLanguages", len(b.lang.slice())+len(b.langNoIndex.slice())) + b.writeConst("NumScripts", len(b.script.slice())) + b.writeConst("NumRegions", len(b.region.slice())) + + // Add dummy codes at the start of each list to represent "unspecified". + b.lang.add("---") + b.script.add("----") + b.region.add("---") + + // common locales + b.locale.parse(meta.DefaultContent.Locales) +} + +// TODO: region inclusion data will probably not be use used in future matchers. + +func (b *builder) computeRegionGroups() { + b.groups = make(map[int]index) + + // Create group indices. + for i := 1; b.region.s[i][0] < 'A'; i++ { // Base M49 indices on regionID. + b.groups[i] = index(len(b.groups)) + } + for _, g := range b.supp.TerritoryContainment.Group { + // Skip UN and EURO zone as they are flattening the containment + // relationship. + if g.Type == "EZ" || g.Type == "UN" { + continue + } + group := b.region.index(g.Type) + if _, ok := b.groups[group]; !ok { + b.groups[group] = index(len(b.groups)) + } + } + if len(b.groups) > 64 { + log.Fatalf("only 64 groups supported, found %d", len(b.groups)) + } + b.writeConst("nRegionGroups", len(b.groups)) +} + +var langConsts = []string{ + "af", "am", "ar", "az", "bg", "bn", "ca", "cs", "da", "de", "el", "en", "es", + "et", "fa", "fi", "fil", "fr", "gu", "he", "hi", "hr", "hu", "hy", "id", "is", + "it", "ja", "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", + "mn", "mo", "mr", "ms", "mul", "my", "nb", "ne", "nl", "no", "pa", "pl", "pt", + "ro", "ru", "sh", "si", "sk", "sl", "sq", "sr", "sv", "sw", "ta", "te", "th", + "tl", "tn", "tr", "uk", "ur", "uz", "vi", "zh", "zu", + + // constants for grandfathered tags (if not already defined) + "jbo", "ami", "bnn", "hak", "tlh", "lb", "nv", "pwn", "tao", "tay", "tsu", + "nn", "sfb", "vgt", "sgg", "cmn", "nan", "hsn", +} + +// writeLanguage generates all tables needed for language canonicalization. +func (b *builder) writeLanguage() { + meta := b.supp.Metadata + + b.writeConst("nonCanonicalUnd", b.lang.index("und")) + b.writeConsts(func(s string) int { return int(b.langIndex(s)) }, langConsts...) + b.writeConst("langPrivateStart", b.langIndex("qaa")) + b.writeConst("langPrivateEnd", b.langIndex("qtz")) + + // Get language codes that need to be mapped (overlong 3-letter codes, + // deprecated 2-letter codes, legacy and grandfathered tags.) + langAliasMap := stringSet{} + aliasTypeMap := map[string]AliasType{} + + // altLangISO3 get the alternative ISO3 names that need to be mapped. + altLangISO3 := stringSet{} + // Add dummy start to avoid the use of index 0. + altLangISO3.add("---") + altLangISO3.updateLater("---", "aa") + + lang := b.lang.clone() + for _, a := range meta.Alias.LanguageAlias { + if a.Replacement == "" { + a.Replacement = "und" + } + // TODO: support mapping to tags + repl := strings.SplitN(a.Replacement, "_", 2)[0] + if a.Reason == "overlong" { + if len(a.Replacement) == 2 && len(a.Type) == 3 { + lang.updateLater(a.Replacement, a.Type) + } + } else if len(a.Type) <= 3 { + switch a.Reason { + case "macrolanguage": + aliasTypeMap[a.Type] = Macro + case "deprecated": + // handled elsewhere + continue + case "bibliographic", "legacy": + if a.Type == "no" { + continue + } + aliasTypeMap[a.Type] = Legacy + default: + log.Fatalf("new %s alias: %s", a.Reason, a.Type) + } + langAliasMap.add(a.Type) + langAliasMap.updateLater(a.Type, repl) + } + } + // Manually add the mapping of "nb" (Norwegian) to its macro language. + // This can be removed if CLDR adopts this change. + langAliasMap.add("nb") + langAliasMap.updateLater("nb", "no") + aliasTypeMap["nb"] = Macro + + for k, v := range b.registry { + // Also add deprecated values for 3-letter ISO codes, which CLDR omits. + if v.typ == "language" && v.deprecated != "" && v.preferred != "" { + langAliasMap.add(k) + langAliasMap.updateLater(k, v.preferred) + aliasTypeMap[k] = Deprecated + } + } + // Fix CLDR mappings. + lang.updateLater("tl", "tgl") + lang.updateLater("sh", "hbs") + lang.updateLater("mo", "mol") + lang.updateLater("no", "nor") + lang.updateLater("tw", "twi") + lang.updateLater("nb", "nob") + lang.updateLater("ak", "aka") + lang.updateLater("bh", "bih") + + // Ensure that each 2-letter code is matched with a 3-letter code. + for _, v := range lang.s[1:] { + s, ok := lang.update[v] + if !ok { + if s, ok = lang.update[langAliasMap.update[v]]; !ok { + continue + } + lang.update[v] = s + } + if v[0] != s[0] { + altLangISO3.add(s) + altLangISO3.updateLater(s, v) + } + } + + // Complete canonicalized language tags. + lang.freeze() + for i, v := range lang.s { + // We can avoid these manual entries by using the IANA registry directly. + // Seems easier to update the list manually, as changes are rare. + // The panic in this loop will trigger if we miss an entry. + add := "" + if s, ok := lang.update[v]; ok { + if s[0] == v[0] { + add = s[1:] + } else { + add = string([]byte{0, byte(altLangISO3.index(s))}) + } + } else if len(v) == 3 { + add = "\x00" + } else { + log.Panicf("no data for long form of %q", v) + } + lang.s[i] += add + } + b.writeConst("lang", tag.Index(lang.join())) + + b.writeConst("langNoIndexOffset", len(b.lang.s)) + + // space of all valid 3-letter language identifiers. + b.writeBitVector("langNoIndex", b.langNoIndex.slice()) + + altLangIndex := []uint16{} + for i, s := range altLangISO3.slice() { + altLangISO3.s[i] += string([]byte{byte(len(altLangIndex))}) + if i > 0 { + idx := b.lang.index(altLangISO3.update[s]) + altLangIndex = append(altLangIndex, uint16(idx)) + } + } + b.writeConst("altLangISO3", tag.Index(altLangISO3.join())) + b.writeSlice("altLangIndex", altLangIndex) + + b.writeSortedMap("AliasMap", &langAliasMap, b.langIndex) + types := make([]AliasType, len(langAliasMap.s)) + for i, s := range langAliasMap.s { + types[i] = aliasTypeMap[s] + } + b.writeSlice("AliasTypes", types) +} + +var scriptConsts = []string{ + "Latn", "Hani", "Hans", "Hant", "Qaaa", "Qaai", "Qabx", "Zinh", "Zyyy", + "Zzzz", +} + +func (b *builder) writeScript() { + b.writeConsts(b.script.index, scriptConsts...) + b.writeConst("script", tag.Index(b.script.join())) + + supp := make([]uint8, len(b.lang.slice())) + for i, v := range b.lang.slice()[1:] { + if sc := b.registry[v].suppressScript; sc != "" { + supp[i+1] = uint8(b.script.index(sc)) + } + } + b.writeSlice("suppressScript", supp) + + // There is only one deprecated script in CLDR. This value is hard-coded. + // We check here if the code must be updated. + for _, a := range b.supp.Metadata.Alias.ScriptAlias { + if a.Type != "Qaai" { + log.Panicf("unexpected deprecated stript %q", a.Type) + } + } +} + +func parseM49(s string) int16 { + if len(s) == 0 { + return 0 + } + v, err := strconv.ParseUint(s, 10, 10) + failOnError(err) + return int16(v) +} + +var regionConsts = []string{ + "001", "419", "BR", "CA", "ES", "GB", "MD", "PT", "UK", "US", + "ZZ", "XA", "XC", "XK", // Unofficial tag for Kosovo. +} + +func (b *builder) writeRegion() { + b.writeConsts(b.region.index, regionConsts...) + + isoOffset := b.region.index("AA") + m49map := make([]int16, len(b.region.slice())) + fromM49map := make(map[int16]int) + altRegionISO3 := "" + altRegionIDs := []uint16{} + + b.writeConst("isoRegionOffset", isoOffset) + + // 2-letter region lookup and mapping to numeric codes. + regionISO := b.region.clone() + regionISO.s = regionISO.s[isoOffset:] + regionISO.sorted = false + + regionTypes := make([]byte, len(b.region.s)) + + // Is the region valid BCP 47? + for s, e := range b.registry { + if len(s) == 2 && s == strings.ToUpper(s) { + i := b.region.index(s) + for _, d := range e.description { + if strings.Contains(d, "Private use") { + regionTypes[i] = iso3166UserAssigned + } + } + regionTypes[i] |= bcp47Region + } + } + + // Is the region a valid ccTLD? + r := gen.OpenIANAFile("domains/root/db") + defer r.Close() + + buf, err := ioutil.ReadAll(r) + failOnError(err) + re := regexp.MustCompile(`"/domains/root/db/([a-z]{2}).html"`) + for _, m := range re.FindAllSubmatch(buf, -1) { + i := b.region.index(strings.ToUpper(string(m[1]))) + regionTypes[i] |= ccTLD + } + + b.writeSlice("regionTypes", regionTypes) + + iso3Set := make(map[string]int) + update := func(iso2, iso3 string) { + i := regionISO.index(iso2) + if j, ok := iso3Set[iso3]; !ok && iso3[0] == iso2[0] { + regionISO.s[i] += iso3[1:] + iso3Set[iso3] = -1 + } else { + if ok && j >= 0 { + regionISO.s[i] += string([]byte{0, byte(j)}) + } else { + iso3Set[iso3] = len(altRegionISO3) + regionISO.s[i] += string([]byte{0, byte(len(altRegionISO3))}) + altRegionISO3 += iso3 + altRegionIDs = append(altRegionIDs, uint16(isoOffset+i)) + } + } + } + for _, tc := range b.supp.CodeMappings.TerritoryCodes { + i := regionISO.index(tc.Type) + isoOffset + if d := m49map[i]; d != 0 { + log.Panicf("%s found as a duplicate UN.M49 code of %03d", tc.Numeric, d) + } + m49 := parseM49(tc.Numeric) + m49map[i] = m49 + if r := fromM49map[m49]; r == 0 { + fromM49map[m49] = i + } else if r != i { + dep := b.registry[regionISO.s[r-isoOffset]].deprecated + if t := b.registry[tc.Type]; t != nil && dep != "" && (t.deprecated == "" || t.deprecated > dep) { + fromM49map[m49] = i + } + } + } + for _, ta := range b.supp.Metadata.Alias.TerritoryAlias { + if len(ta.Type) == 3 && ta.Type[0] <= '9' && len(ta.Replacement) == 2 { + from := parseM49(ta.Type) + if r := fromM49map[from]; r == 0 { + fromM49map[from] = regionISO.index(ta.Replacement) + isoOffset + } + } + } + for _, tc := range b.supp.CodeMappings.TerritoryCodes { + if len(tc.Alpha3) == 3 { + update(tc.Type, tc.Alpha3) + } + } + // This entries are not included in territoryCodes. Mostly 3-letter variants + // of deleted codes and an entry for QU. + for _, m := range []struct{ iso2, iso3 string }{ + {"CT", "CTE"}, + {"DY", "DHY"}, + {"HV", "HVO"}, + {"JT", "JTN"}, + {"MI", "MID"}, + {"NH", "NHB"}, + {"NQ", "ATN"}, + {"PC", "PCI"}, + {"PU", "PUS"}, + {"PZ", "PCZ"}, + {"RH", "RHO"}, + {"VD", "VDR"}, + {"WK", "WAK"}, + // These three-letter codes are used for others as well. + {"FQ", "ATF"}, + } { + update(m.iso2, m.iso3) + } + for i, s := range regionISO.s { + if len(s) != 4 { + regionISO.s[i] = s + " " + } + } + b.writeConst("regionISO", tag.Index(regionISO.join())) + b.writeConst("altRegionISO3", altRegionISO3) + b.writeSlice("altRegionIDs", altRegionIDs) + + // Create list of deprecated regions. + // TODO: consider inserting SF -> FI. Not included by CLDR, but is the only + // Transitionally-reserved mapping not included. + regionOldMap := stringSet{} + // Include regions in territoryAlias (not all are in the IANA registry!) + for _, reg := range b.supp.Metadata.Alias.TerritoryAlias { + if len(reg.Type) == 2 && reg.Reason == "deprecated" && len(reg.Replacement) == 2 { + regionOldMap.add(reg.Type) + regionOldMap.updateLater(reg.Type, reg.Replacement) + i, _ := regionISO.find(reg.Type) + j, _ := regionISO.find(reg.Replacement) + if k := m49map[i+isoOffset]; k == 0 { + m49map[i+isoOffset] = m49map[j+isoOffset] + } + } + } + b.writeSortedMap("regionOldMap", ®ionOldMap, func(s string) uint16 { + return uint16(b.region.index(s)) + }) + // 3-digit region lookup, groupings. + for i := 1; i < isoOffset; i++ { + m := parseM49(b.region.s[i]) + m49map[i] = m + fromM49map[m] = i + } + b.writeSlice("m49", m49map) + + const ( + searchBits = 7 + regionBits = 9 + ) + if len(m49map) >= 1< %d", len(m49map), 1<>searchBits] = int16(len(fromM49)) + } + b.writeSlice("m49Index", m49Index) + b.writeSlice("fromM49", fromM49) +} + +const ( + // TODO: put these lists in regionTypes as user data? Could be used for + // various optimizations and refinements and could be exposed in the API. + iso3166Except = "AC CP DG EA EU FX IC SU TA UK" + iso3166Trans = "AN BU CS NT TP YU ZR" // SF is not in our set of Regions. + // DY and RH are actually not deleted, but indeterminately reserved. + iso3166DelCLDR = "CT DD DY FQ HV JT MI NH NQ PC PU PZ RH VD WK YD" +) + +const ( + iso3166UserAssigned = 1 << iota + ccTLD + bcp47Region +) + +func find(list []string, s string) int { + for i, t := range list { + if t == s { + return i + } + } + return -1 +} + +// writeVariants generates per-variant information and creates a map from variant +// name to index value. We assign index values such that sorting multiple +// variants by index value will result in the correct order. +// There are two types of variants: specialized and general. Specialized variants +// are only applicable to certain language or language-script pairs. Generalized +// variants apply to any language. Generalized variants always sort after +// specialized variants. We will therefore always assign a higher index value +// to a generalized variant than any other variant. Generalized variants are +// sorted alphabetically among themselves. +// Specialized variants may also sort after other specialized variants. Such +// variants will be ordered after any of the variants they may follow. +// We assume that if a variant x is followed by a variant y, then for any prefix +// p of x, p-x is a prefix of y. This allows us to order tags based on the +// maximum of the length of any of its prefixes. +// TODO: it is possible to define a set of Prefix values on variants such that +// a total order cannot be defined to the point that this algorithm breaks. +// In other words, we cannot guarantee the same order of variants for the +// future using the same algorithm or for non-compliant combinations of +// variants. For this reason, consider using simple alphabetic sorting +// of variants and ignore Prefix restrictions altogether. +func (b *builder) writeVariant() { + generalized := stringSet{} + specialized := stringSet{} + specializedExtend := stringSet{} + // Collate the variants by type and check assumptions. + for _, v := range b.variant.slice() { + e := b.registry[v] + if len(e.prefix) == 0 { + generalized.add(v) + continue + } + c := strings.Split(e.prefix[0], "-") + hasScriptOrRegion := false + if len(c) > 1 { + _, hasScriptOrRegion = b.script.find(c[1]) + if !hasScriptOrRegion { + _, hasScriptOrRegion = b.region.find(c[1]) + + } + } + if len(c) == 1 || len(c) == 2 && hasScriptOrRegion { + // Variant is preceded by a language. + specialized.add(v) + continue + } + // Variant is preceded by another variant. + specializedExtend.add(v) + prefix := c[0] + "-" + if hasScriptOrRegion { + prefix += c[1] + } + for _, p := range e.prefix { + // Verify that the prefix minus the last element is a prefix of the + // predecessor element. + i := strings.LastIndex(p, "-") + pred := b.registry[p[i+1:]] + if find(pred.prefix, p[:i]) < 0 { + log.Fatalf("prefix %q for variant %q not consistent with predecessor spec", p, v) + } + // The sorting used below does not work in the general case. It works + // if we assume that variants that may be followed by others only have + // prefixes of the same length. Verify this. + count := strings.Count(p[:i], "-") + for _, q := range pred.prefix { + if c := strings.Count(q, "-"); c != count { + log.Fatalf("variant %q preceding %q has a prefix %q of size %d; want %d", p[i+1:], v, q, c, count) + } + } + if !strings.HasPrefix(p, prefix) { + log.Fatalf("prefix %q of variant %q should start with %q", p, v, prefix) + } + } + } + + // Sort extended variants. + a := specializedExtend.s + less := func(v, w string) bool { + // Sort by the maximum number of elements. + maxCount := func(s string) (max int) { + for _, p := range b.registry[s].prefix { + if c := strings.Count(p, "-"); c > max { + max = c + } + } + return + } + if cv, cw := maxCount(v), maxCount(w); cv != cw { + return cv < cw + } + // Sort by name as tie breaker. + return v < w + } + sort.Sort(funcSorter{less, sort.StringSlice(a)}) + specializedExtend.frozen = true + + // Create index from variant name to index. + variantIndex := make(map[string]uint8) + add := func(s []string) { + for _, v := range s { + variantIndex[v] = uint8(len(variantIndex)) + } + } + add(specialized.slice()) + add(specializedExtend.s) + numSpecialized := len(variantIndex) + add(generalized.slice()) + if n := len(variantIndex); n > 255 { + log.Fatalf("maximum number of variants exceeded: was %d; want <= 255", n) + } + b.writeMap("variantIndex", variantIndex) + b.writeConst("variantNumSpecialized", numSpecialized) +} + +func (b *builder) writeLanguageInfo() { +} + +// writeLikelyData writes tables that are used both for finding parent relations and for +// language matching. Each entry contains additional bits to indicate the status of the +// data to know when it cannot be used for parent relations. +func (b *builder) writeLikelyData() { + const ( + isList = 1 << iota + scriptInFrom + regionInFrom + ) + type ( // generated types + likelyScriptRegion struct { + region uint16 + script uint8 + flags uint8 + } + likelyLangScript struct { + lang uint16 + script uint8 + flags uint8 + } + likelyLangRegion struct { + lang uint16 + region uint16 + } + // likelyTag is used for getting likely tags for group regions, where + // the likely region might be a region contained in the group. + likelyTag struct { + lang uint16 + region uint16 + script uint8 + } + ) + var ( // generated variables + likelyRegionGroup = make([]likelyTag, len(b.groups)) + likelyLang = make([]likelyScriptRegion, len(b.lang.s)) + likelyRegion = make([]likelyLangScript, len(b.region.s)) + likelyScript = make([]likelyLangRegion, len(b.script.s)) + likelyLangList = []likelyScriptRegion{} + likelyRegionList = []likelyLangScript{} + ) + type fromTo struct { + from, to []string + } + langToOther := map[int][]fromTo{} + regionToOther := map[int][]fromTo{} + for _, m := range b.supp.LikelySubtags.LikelySubtag { + from := strings.Split(m.From, "_") + to := strings.Split(m.To, "_") + if len(to) != 3 { + log.Fatalf("invalid number of subtags in %q: found %d, want 3", m.To, len(to)) + } + if len(from) > 3 { + log.Fatalf("invalid number of subtags: found %d, want 1-3", len(from)) + } + if from[0] != to[0] && from[0] != "und" { + log.Fatalf("unexpected language change in expansion: %s -> %s", from, to) + } + if len(from) == 3 { + if from[2] != to[2] { + log.Fatalf("unexpected region change in expansion: %s -> %s", from, to) + } + if from[0] != "und" { + log.Fatalf("unexpected fully specified from tag: %s -> %s", from, to) + } + } + if len(from) == 1 || from[0] != "und" { + id := 0 + if from[0] != "und" { + id = b.lang.index(from[0]) + } + langToOther[id] = append(langToOther[id], fromTo{from, to}) + } else if len(from) == 2 && len(from[1]) == 4 { + sid := b.script.index(from[1]) + likelyScript[sid].lang = uint16(b.langIndex(to[0])) + likelyScript[sid].region = uint16(b.region.index(to[2])) + } else { + r := b.region.index(from[len(from)-1]) + if id, ok := b.groups[r]; ok { + if from[0] != "und" { + log.Fatalf("region changed unexpectedly: %s -> %s", from, to) + } + likelyRegionGroup[id].lang = uint16(b.langIndex(to[0])) + likelyRegionGroup[id].script = uint8(b.script.index(to[1])) + likelyRegionGroup[id].region = uint16(b.region.index(to[2])) + } else { + regionToOther[r] = append(regionToOther[r], fromTo{from, to}) + } + } + } + b.writeType(likelyLangRegion{}) + b.writeSlice("likelyScript", likelyScript) + + for id := range b.lang.s { + list := langToOther[id] + if len(list) == 1 { + likelyLang[id].region = uint16(b.region.index(list[0].to[2])) + likelyLang[id].script = uint8(b.script.index(list[0].to[1])) + } else if len(list) > 1 { + likelyLang[id].flags = isList + likelyLang[id].region = uint16(len(likelyLangList)) + likelyLang[id].script = uint8(len(list)) + for _, x := range list { + flags := uint8(0) + if len(x.from) > 1 { + if x.from[1] == x.to[2] { + flags = regionInFrom + } else { + flags = scriptInFrom + } + } + likelyLangList = append(likelyLangList, likelyScriptRegion{ + region: uint16(b.region.index(x.to[2])), + script: uint8(b.script.index(x.to[1])), + flags: flags, + }) + } + } + } + // TODO: merge suppressScript data with this table. + b.writeType(likelyScriptRegion{}) + b.writeSlice("likelyLang", likelyLang) + b.writeSlice("likelyLangList", likelyLangList) + + for id := range b.region.s { + list := regionToOther[id] + if len(list) == 1 { + likelyRegion[id].lang = uint16(b.langIndex(list[0].to[0])) + likelyRegion[id].script = uint8(b.script.index(list[0].to[1])) + if len(list[0].from) > 2 { + likelyRegion[id].flags = scriptInFrom + } + } else if len(list) > 1 { + likelyRegion[id].flags = isList + likelyRegion[id].lang = uint16(len(likelyRegionList)) + likelyRegion[id].script = uint8(len(list)) + for i, x := range list { + if len(x.from) == 2 && i != 0 || i > 0 && len(x.from) != 3 { + log.Fatalf("unspecified script must be first in list: %v at %d", x.from, i) + } + x := likelyLangScript{ + lang: uint16(b.langIndex(x.to[0])), + script: uint8(b.script.index(x.to[1])), + } + if len(list[0].from) > 2 { + x.flags = scriptInFrom + } + likelyRegionList = append(likelyRegionList, x) + } + } + } + b.writeType(likelyLangScript{}) + b.writeSlice("likelyRegion", likelyRegion) + b.writeSlice("likelyRegionList", likelyRegionList) + + b.writeType(likelyTag{}) + b.writeSlice("likelyRegionGroup", likelyRegionGroup) +} + +func (b *builder) writeRegionInclusionData() { + var ( + // mm holds for each group the set of groups with a distance of 1. + mm = make(map[int][]index) + + // containment holds for each group the transitive closure of + // containment of other groups. + containment = make(map[index][]index) + ) + for _, g := range b.supp.TerritoryContainment.Group { + // Skip UN and EURO zone as they are flattening the containment + // relationship. + if g.Type == "EZ" || g.Type == "UN" { + continue + } + group := b.region.index(g.Type) + groupIdx := b.groups[group] + for _, mem := range strings.Split(g.Contains, " ") { + r := b.region.index(mem) + mm[r] = append(mm[r], groupIdx) + if g, ok := b.groups[r]; ok { + mm[group] = append(mm[group], g) + containment[groupIdx] = append(containment[groupIdx], g) + } + } + } + + regionContainment := make([]uint64, len(b.groups)) + for _, g := range b.groups { + l := containment[g] + + // Compute the transitive closure of containment. + for i := 0; i < len(l); i++ { + l = append(l, containment[l[i]]...) + } + + // Compute the bitmask. + regionContainment[g] = 1 << g + for _, v := range l { + regionContainment[g] |= 1 << v + } + } + b.writeSlice("regionContainment", regionContainment) + + regionInclusion := make([]uint8, len(b.region.s)) + bvs := make(map[uint64]index) + // Make the first bitvector positions correspond with the groups. + for r, i := range b.groups { + bv := uint64(1 << i) + for _, g := range mm[r] { + bv |= 1 << g + } + bvs[bv] = i + regionInclusion[r] = uint8(bvs[bv]) + } + for r := 1; r < len(b.region.s); r++ { + if _, ok := b.groups[r]; !ok { + bv := uint64(0) + for _, g := range mm[r] { + bv |= 1 << g + } + if bv == 0 { + // Pick the world for unspecified regions. + bv = 1 << b.groups[b.region.index("001")] + } + if _, ok := bvs[bv]; !ok { + bvs[bv] = index(len(bvs)) + } + regionInclusion[r] = uint8(bvs[bv]) + } + } + b.writeSlice("regionInclusion", regionInclusion) + regionInclusionBits := make([]uint64, len(bvs)) + for k, v := range bvs { + regionInclusionBits[v] = uint64(k) + } + // Add bit vectors for increasingly large distances until a fixed point is reached. + regionInclusionNext := []uint8{} + for i := 0; i < len(regionInclusionBits); i++ { + bits := regionInclusionBits[i] + next := bits + for i := uint(0); i < uint(len(b.groups)); i++ { + if bits&(1< 6 { + log.Fatalf("Too many groups: %d", i) + } + idToIndex[mv.Id] = uint8(i + 1) + // TODO: also handle '-' + for _, r := range strings.Split(mv.Value, "+") { + todo := []string{r} + for k := 0; k < len(todo); k++ { + r := todo[k] + regionToGroups[b.regionIndex(r)] |= 1 << uint8(i) + todo = append(todo, regionHierarchy[r]...) + } + } + } + b.w.WriteVar("regionToGroups", regionToGroups) + + // maps language id to in- and out-of-group region. + paradigmLocales := [][3]uint16{} + locales := strings.Split(lm[0].ParadigmLocales[0].Locales, " ") + for i := 0; i < len(locales); i += 2 { + x := [3]uint16{} + for j := 0; j < 2; j++ { + pc := strings.SplitN(locales[i+j], "-", 2) + x[0] = b.langIndex(pc[0]) + if len(pc) == 2 { + x[1+j] = uint16(b.regionIndex(pc[1])) + } + } + paradigmLocales = append(paradigmLocales, x) + } + b.w.WriteVar("paradigmLocales", paradigmLocales) + + b.w.WriteType(mutualIntelligibility{}) + b.w.WriteType(scriptIntelligibility{}) + b.w.WriteType(regionIntelligibility{}) + + matchLang := []mutualIntelligibility{} + matchScript := []scriptIntelligibility{} + matchRegion := []regionIntelligibility{} + // Convert the languageMatch entries in lists keyed by desired language. + for _, m := range lm[0].LanguageMatch { + // Different versions of CLDR use different separators. + desired := strings.Replace(m.Desired, "-", "_", -1) + supported := strings.Replace(m.Supported, "-", "_", -1) + d := strings.Split(desired, "_") + s := strings.Split(supported, "_") + if len(d) != len(s) { + log.Fatalf("not supported: desired=%q; supported=%q", desired, supported) + continue + } + distance, _ := strconv.ParseInt(m.Distance, 10, 8) + switch len(d) { + case 2: + if desired == supported && desired == "*_*" { + continue + } + // language-script pair. + matchScript = append(matchScript, scriptIntelligibility{ + wantLang: uint16(b.langIndex(d[0])), + haveLang: uint16(b.langIndex(s[0])), + wantScript: uint8(b.scriptIndex(d[1])), + haveScript: uint8(b.scriptIndex(s[1])), + distance: uint8(distance), + }) + if m.Oneway != "true" { + matchScript = append(matchScript, scriptIntelligibility{ + wantLang: uint16(b.langIndex(s[0])), + haveLang: uint16(b.langIndex(d[0])), + wantScript: uint8(b.scriptIndex(s[1])), + haveScript: uint8(b.scriptIndex(d[1])), + distance: uint8(distance), + }) + } + case 1: + if desired == supported && desired == "*" { + continue + } + if distance == 1 { + // nb == no is already handled by macro mapping. Check there + // really is only this case. + if d[0] != "no" || s[0] != "nb" { + log.Fatalf("unhandled equivalence %s == %s", s[0], d[0]) + } + continue + } + // TODO: consider dropping oneway field and just doubling the entry. + matchLang = append(matchLang, mutualIntelligibility{ + want: uint16(b.langIndex(d[0])), + have: uint16(b.langIndex(s[0])), + distance: uint8(distance), + oneway: m.Oneway == "true", + }) + case 3: + if desired == supported && desired == "*_*_*" { + continue + } + if desired != supported { + // This is now supported by CLDR, but only one case, which + // should already be covered by paradigm locales. For instance, + // test case "und, en, en-GU, en-IN, en-GB ; en-ZA ; en-GB" in + // testdata/CLDRLocaleMatcherTest.txt tests this. + if supported != "en_*_GB" { + log.Fatalf("not supported: desired=%q; supported=%q", desired, supported) + } + continue + } + ri := regionIntelligibility{ + lang: b.langIndex(d[0]), + distance: uint8(distance), + } + if d[1] != "*" { + ri.script = uint8(b.scriptIndex(d[1])) + } + switch { + case d[2] == "*": + ri.group = 0x80 // not contained in anything + case strings.HasPrefix(d[2], "$!"): + ri.group = 0x80 + d[2] = "$" + d[2][len("$!"):] + fallthrough + case strings.HasPrefix(d[2], "$"): + ri.group |= idToIndex[d[2]] + } + matchRegion = append(matchRegion, ri) + default: + log.Fatalf("not supported: desired=%q; supported=%q", desired, supported) + } + } + sort.SliceStable(matchLang, func(i, j int) bool { + return matchLang[i].distance < matchLang[j].distance + }) + b.w.WriteComment(` + matchLang holds pairs of langIDs of base languages that are typically + mutually intelligible. Each pair is associated with a confidence and + whether the intelligibility goes one or both ways.`) + b.w.WriteVar("matchLang", matchLang) + + b.w.WriteComment(` + matchScript holds pairs of scriptIDs where readers of one script + can typically also read the other. Each is associated with a confidence.`) + sort.SliceStable(matchScript, func(i, j int) bool { + return matchScript[i].distance < matchScript[j].distance + }) + b.w.WriteVar("matchScript", matchScript) + + sort.SliceStable(matchRegion, func(i, j int) bool { + return matchRegion[i].distance < matchRegion[j].distance + }) + b.w.WriteVar("matchRegion", matchRegion) +} diff --git a/vendor/golang.org/x/text/unicode/bidi/gen.go b/vendor/golang.org/x/text/unicode/bidi/gen.go new file mode 100644 index 0000000000..987fc169cc --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen.go @@ -0,0 +1,133 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "flag" + "log" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/triegen" + "golang.org/x/text/internal/ucd" +) + +var outputFile = flag.String("out", "tables.go", "output file") + +func main() { + gen.Init() + gen.Repackage("gen_trieval.go", "trieval.go", "bidi") + gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi") + + genTables() +} + +// bidiClass names and codes taken from class "bc" in +// https://www.unicode.org/Public/8.0.0/ucd/PropertyValueAliases.txt +var bidiClass = map[string]Class{ + "AL": AL, // ArabicLetter + "AN": AN, // ArabicNumber + "B": B, // ParagraphSeparator + "BN": BN, // BoundaryNeutral + "CS": CS, // CommonSeparator + "EN": EN, // EuropeanNumber + "ES": ES, // EuropeanSeparator + "ET": ET, // EuropeanTerminator + "L": L, // LeftToRight + "NSM": NSM, // NonspacingMark + "ON": ON, // OtherNeutral + "R": R, // RightToLeft + "S": S, // SegmentSeparator + "WS": WS, // WhiteSpace + + "FSI": Control, + "PDF": Control, + "PDI": Control, + "LRE": Control, + "LRI": Control, + "LRO": Control, + "RLE": Control, + "RLI": Control, + "RLO": Control, +} + +func genTables() { + if numClass > 0x0F { + log.Fatalf("Too many Class constants (%#x > 0x0F).", numClass) + } + w := gen.NewCodeWriter() + defer w.WriteVersionedGoFile(*outputFile, "bidi") + + gen.WriteUnicodeVersion(w) + + t := triegen.NewTrie("bidi") + + // Build data about bracket mapping. These bits need to be or-ed with + // any other bits. + orMask := map[rune]uint64{} + + xorMap := map[rune]int{} + xorMasks := []rune{0} // First value is no-op. + + ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) { + r1 := p.Rune(0) + r2 := p.Rune(1) + xor := r1 ^ r2 + if _, ok := xorMap[xor]; !ok { + xorMap[xor] = len(xorMasks) + xorMasks = append(xorMasks, xor) + } + entry := uint64(xorMap[xor]) << xorMaskShift + switch p.String(2) { + case "o": + entry |= openMask + case "c", "n": + default: + log.Fatalf("Unknown bracket class %q.", p.String(2)) + } + orMask[r1] = entry + }) + + w.WriteComment(` + xorMasks contains masks to be xor-ed with brackets to get the reverse + version.`) + w.WriteVar("xorMasks", xorMasks) + + done := map[rune]bool{} + + insert := func(r rune, c Class) { + if !done[r] { + t.Insert(r, orMask[r]|uint64(c)) + done[r] = true + } + } + + // Insert the derived BiDi properties. + ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) { + r := p.Rune(0) + class, ok := bidiClass[p.String(1)] + if !ok { + log.Fatalf("%U: Unknown BiDi class %q", r, p.String(1)) + } + insert(r, class) + }) + visitDefaults(insert) + + // TODO: use sparse blocks. This would reduce table size considerably + // from the looks of it. + + sz, err := t.Gen(w) + if err != nil { + log.Fatal(err) + } + w.Size += sz +} + +// dummy values to make methods in gen_common compile. The real versions +// will be generated by this file to tables.go. +var ( + xorMasks []rune +) diff --git a/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go b/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go new file mode 100644 index 0000000000..02c3b505d6 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go @@ -0,0 +1,57 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "unicode" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/ucd" + "golang.org/x/text/unicode/rangetable" +) + +// These tables are hand-extracted from: +// https://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt +func visitDefaults(fn func(r rune, c Class)) { + // first write default values for ranges listed above. + visitRunes(fn, AL, []rune{ + 0x0600, 0x07BF, // Arabic + 0x08A0, 0x08FF, // Arabic Extended-A + 0xFB50, 0xFDCF, // Arabic Presentation Forms + 0xFDF0, 0xFDFF, + 0xFE70, 0xFEFF, + 0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols + }) + visitRunes(fn, R, []rune{ + 0x0590, 0x05FF, // Hebrew + 0x07C0, 0x089F, // Nko et al. + 0xFB1D, 0xFB4F, + 0x00010800, 0x00010FFF, // Cypriot Syllabary et. al. + 0x0001E800, 0x0001EDFF, + 0x0001EF00, 0x0001EFFF, + }) + visitRunes(fn, ET, []rune{ // European Terminator + 0x20A0, 0x20Cf, // Currency symbols + }) + rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) { + fn(r, BN) // Boundary Neutral + }) + ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) { + if p.String(1) == "Default_Ignorable_Code_Point" { + fn(p.Rune(0), BN) // Boundary Neutral + } + }) +} + +func visitRunes(fn func(r rune, c Class), c Class, runes []rune) { + for i := 0; i < len(runes); i += 2 { + lo, hi := runes[i], runes[i+1] + for j := lo; j <= hi; j++ { + fn(j, c) + } + } +} diff --git a/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go b/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go new file mode 100644 index 0000000000..9cb9942894 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go @@ -0,0 +1,64 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// Class is the Unicode BiDi class. Each rune has a single class. +type Class uint + +const ( + L Class = iota // LeftToRight + R // RightToLeft + EN // EuropeanNumber + ES // EuropeanSeparator + ET // EuropeanTerminator + AN // ArabicNumber + CS // CommonSeparator + B // ParagraphSeparator + S // SegmentSeparator + WS // WhiteSpace + ON // OtherNeutral + BN // BoundaryNeutral + NSM // NonspacingMark + AL // ArabicLetter + Control // Control LRO - PDI + + numClass + + LRO // LeftToRightOverride + RLO // RightToLeftOverride + LRE // LeftToRightEmbedding + RLE // RightToLeftEmbedding + PDF // PopDirectionalFormat + LRI // LeftToRightIsolate + RLI // RightToLeftIsolate + FSI // FirstStrongIsolate + PDI // PopDirectionalIsolate + + unknownClass = ^Class(0) +) + +var controlToClass = map[rune]Class{ + 0x202D: LRO, // LeftToRightOverride, + 0x202E: RLO, // RightToLeftOverride, + 0x202A: LRE, // LeftToRightEmbedding, + 0x202B: RLE, // RightToLeftEmbedding, + 0x202C: PDF, // PopDirectionalFormat, + 0x2066: LRI, // LeftToRightIsolate, + 0x2067: RLI, // RightToLeftIsolate, + 0x2068: FSI, // FirstStrongIsolate, + 0x2069: PDI, // PopDirectionalIsolate, +} + +// A trie entry has the following bits: +// 7..5 XOR mask for brackets +// 4 1: Bracket open, 0: Bracket close +// 3..0 Class type + +const ( + openMask = 0x10 + xorMaskShift = 5 +) diff --git a/vendor/golang.org/x/text/unicode/norm/maketables.go b/vendor/golang.org/x/text/unicode/norm/maketables.go new file mode 100644 index 0000000000..30a3aa9334 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/maketables.go @@ -0,0 +1,986 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Normalization table generator. +// Data read from the web. +// See forminfo.go for a description of the trie values associated with each rune. + +package main + +import ( + "bytes" + "encoding/binary" + "flag" + "fmt" + "io" + "log" + "sort" + "strconv" + "strings" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/triegen" + "golang.org/x/text/internal/ucd" +) + +func main() { + gen.Init() + loadUnicodeData() + compactCCC() + loadCompositionExclusions() + completeCharFields(FCanonical) + completeCharFields(FCompatibility) + computeNonStarterCounts() + verifyComputed() + printChars() + testDerived() + printTestdata() + makeTables() +} + +var ( + tablelist = flag.String("tables", + "all", + "comma-separated list of which tables to generate; "+ + "can be 'decomp', 'recomp', 'info' and 'all'") + test = flag.Bool("test", + false, + "test existing tables against DerivedNormalizationProps and generate test data for regression testing") + verbose = flag.Bool("verbose", + false, + "write data to stdout as it is parsed") +) + +const MaxChar = 0x10FFFF // anything above this shouldn't exist + +// Quick Check properties of runes allow us to quickly +// determine whether a rune may occur in a normal form. +// For a given normal form, a rune may be guaranteed to occur +// verbatim (QC=Yes), may or may not combine with another +// rune (QC=Maybe), or may not occur (QC=No). +type QCResult int + +const ( + QCUnknown QCResult = iota + QCYes + QCNo + QCMaybe +) + +func (r QCResult) String() string { + switch r { + case QCYes: + return "Yes" + case QCNo: + return "No" + case QCMaybe: + return "Maybe" + } + return "***UNKNOWN***" +} + +const ( + FCanonical = iota // NFC or NFD + FCompatibility // NFKC or NFKD + FNumberOfFormTypes +) + +const ( + MComposed = iota // NFC or NFKC + MDecomposed // NFD or NFKD + MNumberOfModes +) + +// This contains only the properties we're interested in. +type Char struct { + name string + codePoint rune // if zero, this index is not a valid code point. + ccc uint8 // canonical combining class + origCCC uint8 + excludeInComp bool // from CompositionExclusions.txt + compatDecomp bool // it has a compatibility expansion + + nTrailingNonStarters uint8 + nLeadingNonStarters uint8 // must be equal to trailing if non-zero + + forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility + + state State +} + +var chars = make([]Char, MaxChar+1) +var cccMap = make(map[uint8]uint8) + +func (c Char) String() string { + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name) + fmt.Fprintf(buf, " ccc: %v\n", c.ccc) + fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp) + fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp) + fmt.Fprintf(buf, " state: %v\n", c.state) + fmt.Fprintf(buf, " NFC:\n") + fmt.Fprint(buf, c.forms[FCanonical]) + fmt.Fprintf(buf, " NFKC:\n") + fmt.Fprint(buf, c.forms[FCompatibility]) + + return buf.String() +} + +// In UnicodeData.txt, some ranges are marked like this: +// 3400;;Lo;0;L;;;;;N;;;;; +// 4DB5;;Lo;0;L;;;;;N;;;;; +// parseCharacter keeps a state variable indicating the weirdness. +type State int + +const ( + SNormal State = iota // known to be zero for the type + SFirst + SLast + SMissing +) + +var lastChar = rune('\u0000') + +func (c Char) isValid() bool { + return c.codePoint != 0 && c.state != SMissing +} + +type FormInfo struct { + quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed + verified [MNumberOfModes]bool // index: MComposed or MDecomposed + + combinesForward bool // May combine with rune on the right + combinesBackward bool // May combine with rune on the left + isOneWay bool // Never appears in result + inDecomp bool // Some decompositions result in this char. + decomp Decomposition + expandedDecomp Decomposition +} + +func (f FormInfo) String() string { + buf := bytes.NewBuffer(make([]byte, 0)) + + fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed]) + fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed]) + fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward) + fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward) + fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay) + fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp) + fmt.Fprintf(buf, " decomposition: %X\n", f.decomp) + fmt.Fprintf(buf, " expandedDecomp: %X\n", f.expandedDecomp) + + return buf.String() +} + +type Decomposition []rune + +func parseDecomposition(s string, skipfirst bool) (a []rune, err error) { + decomp := strings.Split(s, " ") + if len(decomp) > 0 && skipfirst { + decomp = decomp[1:] + } + for _, d := range decomp { + point, err := strconv.ParseUint(d, 16, 64) + if err != nil { + return a, err + } + a = append(a, rune(point)) + } + return a, nil +} + +func loadUnicodeData() { + f := gen.OpenUCDFile("UnicodeData.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + r := p.Rune(ucd.CodePoint) + char := &chars[r] + + char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass)) + decmap := p.String(ucd.DecompMapping) + + exp, err := parseDecomposition(decmap, false) + isCompat := false + if err != nil { + if len(decmap) > 0 { + exp, err = parseDecomposition(decmap, true) + if err != nil { + log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err) + } + isCompat = true + } + } + + char.name = p.String(ucd.Name) + char.codePoint = r + char.forms[FCompatibility].decomp = exp + if !isCompat { + char.forms[FCanonical].decomp = exp + } else { + char.compatDecomp = true + } + if len(decmap) > 0 { + char.forms[FCompatibility].decomp = exp + } + } + if err := p.Err(); err != nil { + log.Fatal(err) + } +} + +// compactCCC converts the sparse set of CCC values to a continguous one, +// reducing the number of bits needed from 8 to 6. +func compactCCC() { + m := make(map[uint8]uint8) + for i := range chars { + c := &chars[i] + m[c.ccc] = 0 + } + cccs := []int{} + for v, _ := range m { + cccs = append(cccs, int(v)) + } + sort.Ints(cccs) + for i, c := range cccs { + cccMap[uint8(i)] = uint8(c) + m[uint8(c)] = uint8(i) + } + for i := range chars { + c := &chars[i] + c.origCCC = c.ccc + c.ccc = m[c.ccc] + } + if len(m) >= 1<<6 { + log.Fatalf("too many difference CCC values: %d >= 64", len(m)) + } +} + +// CompositionExclusions.txt has form: +// 0958 # ... +// See https://unicode.org/reports/tr44/ for full explanation +func loadCompositionExclusions() { + f := gen.OpenUCDFile("CompositionExclusions.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + c := &chars[p.Rune(0)] + if c.excludeInComp { + log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) + } + c.excludeInComp = true + } + if e := p.Err(); e != nil { + log.Fatal(e) + } +} + +// hasCompatDecomp returns true if any of the recursive +// decompositions contains a compatibility expansion. +// In this case, the character may not occur in NFK*. +func hasCompatDecomp(r rune) bool { + c := &chars[r] + if c.compatDecomp { + return true + } + for _, d := range c.forms[FCompatibility].decomp { + if hasCompatDecomp(d) { + return true + } + } + return false +} + +// Hangul related constants. +const ( + HangulBase = 0xAC00 + HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28) + + JamoLBase = 0x1100 + JamoLEnd = 0x1113 + JamoVBase = 0x1161 + JamoVEnd = 0x1176 + JamoTBase = 0x11A8 + JamoTEnd = 0x11C3 + + JamoLVTCount = 19 * 21 * 28 + JamoTCount = 28 +) + +func isHangul(r rune) bool { + return HangulBase <= r && r < HangulEnd +} + +func isHangulWithoutJamoT(r rune) bool { + if !isHangul(r) { + return false + } + r -= HangulBase + return r < JamoLVTCount && r%JamoTCount == 0 +} + +func ccc(r rune) uint8 { + return chars[r].ccc +} + +// Insert a rune in a buffer, ordered by Canonical Combining Class. +func insertOrdered(b Decomposition, r rune) Decomposition { + n := len(b) + b = append(b, 0) + cc := ccc(r) + if cc > 0 { + // Use bubble sort. + for ; n > 0; n-- { + if ccc(b[n-1]) <= cc { + break + } + b[n] = b[n-1] + } + } + b[n] = r + return b +} + +// Recursively decompose. +func decomposeRecursive(form int, r rune, d Decomposition) Decomposition { + dcomp := chars[r].forms[form].decomp + if len(dcomp) == 0 { + return insertOrdered(d, r) + } + for _, c := range dcomp { + d = decomposeRecursive(form, c, d) + } + return d +} + +func completeCharFields(form int) { + // Phase 0: pre-expand decomposition. + for i := range chars { + f := &chars[i].forms[form] + if len(f.decomp) == 0 { + continue + } + exp := make(Decomposition, 0) + for _, c := range f.decomp { + exp = decomposeRecursive(form, c, exp) + } + f.expandedDecomp = exp + } + + // Phase 1: composition exclusion, mark decomposition. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + // Marks script-specific exclusions and version restricted. + f.isOneWay = c.excludeInComp + + // Singletons + f.isOneWay = f.isOneWay || len(f.decomp) == 1 + + // Non-starter decompositions + if len(f.decomp) > 1 { + chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0 + f.isOneWay = f.isOneWay || chk + } + + // Runes that decompose into more than two runes. + f.isOneWay = f.isOneWay || len(f.decomp) > 2 + + if form == FCompatibility { + f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint) + } + + for _, r := range f.decomp { + chars[r].forms[form].inDecomp = true + } + } + + // Phase 2: forward and backward combining. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + if !f.isOneWay && len(f.decomp) == 2 { + f0 := &chars[f.decomp[0]].forms[form] + f1 := &chars[f.decomp[1]].forms[form] + if !f0.isOneWay { + f0.combinesForward = true + } + if !f1.isOneWay { + f1.combinesBackward = true + } + } + if isHangulWithoutJamoT(rune(i)) { + f.combinesForward = true + } + } + + // Phase 3: quick check values. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + switch { + case len(f.decomp) > 0: + f.quickCheck[MDecomposed] = QCNo + case isHangul(rune(i)): + f.quickCheck[MDecomposed] = QCNo + default: + f.quickCheck[MDecomposed] = QCYes + } + switch { + case f.isOneWay: + f.quickCheck[MComposed] = QCNo + case (i & 0xffff00) == JamoLBase: + f.quickCheck[MComposed] = QCYes + if JamoLBase <= i && i < JamoLEnd { + f.combinesForward = true + } + if JamoVBase <= i && i < JamoVEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + f.combinesForward = true + } + if JamoTBase <= i && i < JamoTEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + } + case !f.combinesBackward: + f.quickCheck[MComposed] = QCYes + default: + f.quickCheck[MComposed] = QCMaybe + } + } +} + +func computeNonStarterCounts() { + // Phase 4: leading and trailing non-starter count + for i := range chars { + c := &chars[i] + + runes := []rune{rune(i)} + // We always use FCompatibility so that the CGJ insertion points do not + // change for repeated normalizations with different forms. + if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 { + runes = exp + } + // We consider runes that combine backwards to be non-starters for the + // purpose of Stream-Safe Text Processing. + for _, r := range runes { + if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { + break + } + c.nLeadingNonStarters++ + } + for i := len(runes) - 1; i >= 0; i-- { + if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { + break + } + c.nTrailingNonStarters++ + } + if c.nTrailingNonStarters > 3 { + log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes) + } + + if isHangul(rune(i)) { + c.nTrailingNonStarters = 2 + if isHangulWithoutJamoT(rune(i)) { + c.nTrailingNonStarters = 1 + } + } + + if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t { + log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t) + } + if t := c.nTrailingNonStarters; t > 3 { + log.Fatalf("%U: number of trailing non-starters is %d > 3", t) + } + } +} + +func printBytes(w io.Writer, b []byte, name string) { + fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b)) + fmt.Fprintf(w, "var %s = [...]byte {", name) + for i, c := range b { + switch { + case i%64 == 0: + fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63) + case i%8 == 0: + fmt.Fprintf(w, "\n") + } + fmt.Fprintf(w, "0x%.2X, ", c) + } + fmt.Fprint(w, "\n}\n\n") +} + +// See forminfo.go for format. +func makeEntry(f *FormInfo, c *Char) uint16 { + e := uint16(0) + if r := c.codePoint; HangulBase <= r && r < HangulEnd { + e |= 0x40 + } + if f.combinesForward { + e |= 0x20 + } + if f.quickCheck[MDecomposed] == QCNo { + e |= 0x4 + } + switch f.quickCheck[MComposed] { + case QCYes: + case QCNo: + e |= 0x10 + case QCMaybe: + e |= 0x18 + default: + log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed]) + } + e |= uint16(c.nTrailingNonStarters) + return e +} + +// decompSet keeps track of unique decompositions, grouped by whether +// the decomposition is followed by a trailing and/or leading CCC. +type decompSet [7]map[string]bool + +const ( + normalDecomp = iota + firstMulti + firstCCC + endMulti + firstLeadingCCC + firstCCCZeroExcept + firstStarterWithNLead + lastDecomp +) + +var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"} + +func makeDecompSet() decompSet { + m := decompSet{} + for i := range m { + m[i] = make(map[string]bool) + } + return m +} +func (m *decompSet) insert(key int, s string) { + m[key][s] = true +} + +func printCharInfoTables(w io.Writer) int { + mkstr := func(r rune, f *FormInfo) (int, string) { + d := f.expandedDecomp + s := string([]rune(d)) + if max := 1 << 6; len(s) >= max { + const msg = "%U: too many bytes in decomposition: %d >= %d" + log.Fatalf(msg, r, len(s), max) + } + head := uint8(len(s)) + if f.quickCheck[MComposed] != QCYes { + head |= 0x40 + } + if f.combinesForward { + head |= 0x80 + } + s = string([]byte{head}) + s + + lccc := ccc(d[0]) + tccc := ccc(d[len(d)-1]) + cc := ccc(r) + if cc != 0 && lccc == 0 && tccc == 0 { + log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc) + } + if tccc < lccc && lccc != 0 { + const msg = "%U: lccc (%d) must be <= tcc (%d)" + log.Fatalf(msg, r, lccc, tccc) + } + index := normalDecomp + nTrail := chars[r].nTrailingNonStarters + nLead := chars[r].nLeadingNonStarters + if tccc > 0 || lccc > 0 || nTrail > 0 { + tccc <<= 2 + tccc |= nTrail + s += string([]byte{tccc}) + index = endMulti + for _, r := range d[1:] { + if ccc(r) == 0 { + index = firstCCC + } + } + if lccc > 0 || nLead > 0 { + s += string([]byte{lccc}) + if index == firstCCC { + log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r) + } + index = firstLeadingCCC + } + if cc != lccc { + if cc != 0 { + log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc) + } + index = firstCCCZeroExcept + } + } else if len(d) > 1 { + index = firstMulti + } + return index, s + } + + decompSet := makeDecompSet() + const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail. + decompSet.insert(firstStarterWithNLead, nLeadStr) + + // Store the uniqued decompositions in a byte buffer, + // preceded by their byte length. + for _, c := range chars { + for _, f := range c.forms { + if len(f.expandedDecomp) == 0 { + continue + } + if f.combinesBackward { + log.Fatalf("%U: combinesBackward and decompose", c.codePoint) + } + index, s := mkstr(c.codePoint, &f) + decompSet.insert(index, s) + } + } + + decompositions := bytes.NewBuffer(make([]byte, 0, 10000)) + size := 0 + positionMap := make(map[string]uint16) + decompositions.WriteString("\000") + fmt.Fprintln(w, "const (") + for i, m := range decompSet { + sa := []string{} + for s := range m { + sa = append(sa, s) + } + sort.Strings(sa) + for _, s := range sa { + p := decompositions.Len() + decompositions.WriteString(s) + positionMap[s] = uint16(p) + } + if cname[i] != "" { + fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len()) + } + } + fmt.Fprintln(w, "maxDecomp = 0x8000") + fmt.Fprintln(w, ")") + b := decompositions.Bytes() + printBytes(w, b, "decomps") + size += len(b) + + varnames := []string{"nfc", "nfkc"} + for i := 0; i < FNumberOfFormTypes; i++ { + trie := triegen.NewTrie(varnames[i]) + + for r, c := range chars { + f := c.forms[i] + d := f.expandedDecomp + if len(d) != 0 { + _, key := mkstr(c.codePoint, &f) + trie.Insert(rune(r), uint64(positionMap[key])) + if c.ccc != ccc(d[0]) { + // We assume the lead ccc of a decomposition !=0 in this case. + if ccc(d[0]) == 0 { + log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc) + } + } + } else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward { + // Handle cases where it can't be detected that the nLead should be equal + // to nTrail. + trie.Insert(c.codePoint, uint64(positionMap[nLeadStr])) + } else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 { + trie.Insert(c.codePoint, uint64(0x8000|v)) + } + } + sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]})) + if err != nil { + log.Fatal(err) + } + size += sz + } + return size +} + +func contains(sa []string, s string) bool { + for _, a := range sa { + if a == s { + return true + } + } + return false +} + +func makeTables() { + w := &bytes.Buffer{} + + size := 0 + if *tablelist == "" { + return + } + list := strings.Split(*tablelist, ",") + if *tablelist == "all" { + list = []string{"recomp", "info"} + } + + // Compute maximum decomposition size. + max := 0 + for _, c := range chars { + if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max { + max = n + } + } + fmt.Fprintln(w, `import "sync"`) + fmt.Fprintln(w) + + fmt.Fprintln(w, "const (") + fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.") + fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion()) + fmt.Fprintln(w) + fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform") + fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at") + fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that") + fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.") + fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max) + fmt.Fprintln(w, ")\n") + + // Print the CCC remap table. + size += len(cccMap) + fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap)) + for i := 0; i < len(cccMap); i++ { + if i%8 == 0 { + fmt.Fprintln(w) + } + fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)]) + } + fmt.Fprintln(w, "\n}\n") + + if contains(list, "info") { + size += printCharInfoTables(w) + } + + if contains(list, "recomp") { + // Note that we use 32 bit keys, instead of 64 bit. + // This clips the bits of three entries, but we know + // this won't cause a collision. The compiler will catch + // any changes made to UnicodeData.txt that introduces + // a collision. + // Note that the recomposition map for NFC and NFKC + // are identical. + + // Recomposition map + nrentries := 0 + for _, c := range chars { + f := c.forms[FCanonical] + if !f.isOneWay && len(f.decomp) > 0 { + nrentries++ + } + } + sz := nrentries * 8 + size += sz + fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz) + fmt.Fprintln(w, "var recompMap map[uint32]rune") + fmt.Fprintln(w, "var recompMapOnce sync.Once\n") + fmt.Fprintln(w, `const recompMapPacked = "" +`) + var buf [8]byte + for i, c := range chars { + f := c.forms[FCanonical] + d := f.decomp + if !f.isOneWay && len(d) > 0 { + key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) + binary.BigEndian.PutUint32(buf[:4], key) + binary.BigEndian.PutUint32(buf[4:], uint32(i)) + fmt.Fprintf(w, "\t\t%q + // 0x%.8X: 0x%.8X\n", string(buf[:]), key, uint32(i)) + } + } + // hack so we don't have to special case the trailing plus sign + fmt.Fprintf(w, ` ""`) + fmt.Fprintln(w) + } + + fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) + gen.WriteVersionedGoFile("tables.go", "norm", w.Bytes()) +} + +func printChars() { + if *verbose { + for _, c := range chars { + if !c.isValid() || c.state == SMissing { + continue + } + fmt.Println(c) + } + } +} + +// verifyComputed does various consistency tests. +func verifyComputed() { + for i, c := range chars { + for _, f := range c.forms { + isNo := (f.quickCheck[MDecomposed] == QCNo) + if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) { + log.Fatalf("%U: NF*D QC must be No if rune decomposes", i) + } + + isMaybe := f.quickCheck[MComposed] == QCMaybe + if f.combinesBackward != isMaybe { + log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i) + } + if len(f.decomp) > 0 && f.combinesForward && isMaybe { + log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i) + } + + if len(f.expandedDecomp) != 0 { + continue + } + if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b { + // We accept these runes to be treated differently (it only affects + // segment breaking in iteration, most likely on improper use), but + // reconsider if more characters are added. + // U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L; 3099;;;;N;;;;; + // U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L; 309A;;;;N;;;;; + // U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;; + // U+318E HANGUL LETTER ARAEAE;Lo;0;L; 11A1;;;;N;HANGUL LETTER ALAE AE;;;; + // U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;; + // U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L; 3163;;;;N;;;;; + if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) { + log.Fatalf("%U: nLead was %v; want %v", i, a, b) + } + } + } + nfc := c.forms[FCanonical] + nfkc := c.forms[FCompatibility] + if nfc.combinesBackward != nfkc.combinesBackward { + log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint) + } + } +} + +// Use values in DerivedNormalizationProps.txt to compare against the +// values we computed. +// DerivedNormalizationProps.txt has form: +// 00C0..00C5 ; NFD_QC; N # ... +// 0374 ; NFD_QC; N # ... +// See https://unicode.org/reports/tr44/ for full explanation +func testDerived() { + f := gen.OpenUCDFile("DerivedNormalizationProps.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + r := p.Rune(0) + c := &chars[r] + + var ftype, mode int + qt := p.String(1) + switch qt { + case "NFC_QC": + ftype, mode = FCanonical, MComposed + case "NFD_QC": + ftype, mode = FCanonical, MDecomposed + case "NFKC_QC": + ftype, mode = FCompatibility, MComposed + case "NFKD_QC": + ftype, mode = FCompatibility, MDecomposed + default: + continue + } + var qr QCResult + switch p.String(2) { + case "Y": + qr = QCYes + case "N": + qr = QCNo + case "M": + qr = QCMaybe + default: + log.Fatalf(`Unexpected quick check value "%s"`, p.String(2)) + } + if got := c.forms[ftype].quickCheck[mode]; got != qr { + log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr) + } + c.forms[ftype].verified[mode] = true + } + if err := p.Err(); err != nil { + log.Fatal(err) + } + // Any unspecified value must be QCYes. Verify this. + for i, c := range chars { + for j, fd := range c.forms { + for k, qr := range fd.quickCheck { + if !fd.verified[k] && qr != QCYes { + m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" + log.Printf(m, i, j, k, qr, c.name) + } + } + } + } +} + +var testHeader = `const ( + Yes = iota + No + Maybe +) + +type formData struct { + qc uint8 + combinesForward bool + decomposition string +} + +type runeData struct { + r rune + ccc uint8 + nLead uint8 + nTrail uint8 + f [2]formData // 0: canonical; 1: compatibility +} + +func f(qc uint8, cf bool, dec string) [2]formData { + return [2]formData{{qc, cf, dec}, {qc, cf, dec}} +} + +func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData { + return [2]formData{{qc, cf, d}, {qck, cfk, dk}} +} + +var testData = []runeData{ +` + +func printTestdata() { + type lastInfo struct { + ccc uint8 + nLead uint8 + nTrail uint8 + f string + } + + last := lastInfo{} + w := &bytes.Buffer{} + fmt.Fprintf(w, testHeader) + for r, c := range chars { + f := c.forms[FCanonical] + qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) + f = c.forms[FCompatibility] + qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) + s := "" + if d == dk && qc == qck && cf == cfk { + s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d) + } else { + s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk) + } + current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s} + if last != current { + fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s) + last = current + } + } + fmt.Fprintln(w, "}") + gen.WriteVersionedGoFile("data_test.go", "norm", w.Bytes()) +} diff --git a/vendor/golang.org/x/text/unicode/norm/triegen.go b/vendor/golang.org/x/text/unicode/norm/triegen.go new file mode 100644 index 0000000000..45d711900d --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/triegen.go @@ -0,0 +1,117 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Trie table generator. +// Used by make*tables tools to generate a go file with trie data structures +// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte +// sequence are used to lookup offsets in the index table to be used for the +// next byte. The last byte is used to index into a table with 16-bit values. + +package main + +import ( + "fmt" + "io" +) + +const maxSparseEntries = 16 + +type normCompacter struct { + sparseBlocks [][]uint64 + sparseOffset []uint16 + sparseCount int + name string +} + +func mostFrequentStride(a []uint64) int { + counts := make(map[int]int) + var v int + for _, x := range a { + if stride := int(x) - v; v != 0 && stride >= 0 { + counts[stride]++ + } + v = int(x) + } + var maxs, maxc int + for stride, cnt := range counts { + if cnt > maxc || (cnt == maxc && stride < maxs) { + maxs, maxc = stride, cnt + } + } + return maxs +} + +func countSparseEntries(a []uint64) int { + stride := mostFrequentStride(a) + var v, count int + for _, tv := range a { + if int(tv)-v != stride { + if tv != 0 { + count++ + } + } + v = int(tv) + } + return count +} + +func (c *normCompacter) Size(v []uint64) (sz int, ok bool) { + if n := countSparseEntries(v); n <= maxSparseEntries { + return (n+1)*4 + 2, true + } + return 0, false +} + +func (c *normCompacter) Store(v []uint64) uint32 { + h := uint32(len(c.sparseOffset)) + c.sparseBlocks = append(c.sparseBlocks, v) + c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount)) + c.sparseCount += countSparseEntries(v) + 1 + return h +} + +func (c *normCompacter) Handler() string { + return c.name + "Sparse.lookup" +} + +func (c *normCompacter) Print(w io.Writer) (retErr error) { + p := func(f string, x ...interface{}) { + if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil { + retErr = err + } + } + + ls := len(c.sparseBlocks) + p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2) + p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset) + + ns := c.sparseCount + p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4) + p("var %sSparseValues = [%d]valueRange {", c.name, ns) + for i, b := range c.sparseBlocks { + p("\n// Block %#x, offset %#x", i, c.sparseOffset[i]) + var v int + stride := mostFrequentStride(b) + n := countSparseEntries(b) + p("\n{value:%#04x,lo:%#02x},", stride, uint8(n)) + for i, nv := range b { + if int(nv)-v != stride { + if v != 0 { + p(",hi:%#02x},", 0x80+i-1) + } + if nv != 0 { + p("\n{value:%#04x,lo:%#02x", nv, 0x80+i) + } + } + v = int(nv) + } + if v != 0 { + p(",hi:%#02x},", 0x80+len(b)-1) + } + } + p("\n}\n\n") + return +} diff --git a/vendor/golang.org/x/text/width/gen.go b/vendor/golang.org/x/text/width/gen.go new file mode 100644 index 0000000000..092277e1f6 --- /dev/null +++ b/vendor/golang.org/x/text/width/gen.go @@ -0,0 +1,115 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// This program generates the trie for width operations. The generated table +// includes width category information as well as the normalization mappings. +package main + +import ( + "bytes" + "fmt" + "io" + "log" + "math" + "unicode/utf8" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/triegen" +) + +// See gen_common.go for flags. + +func main() { + gen.Init() + genTables() + genTests() + gen.Repackage("gen_trieval.go", "trieval.go", "width") + gen.Repackage("gen_common.go", "common_test.go", "width") +} + +func genTables() { + t := triegen.NewTrie("width") + // fold and inverse mappings. See mapComment for a description of the format + // of each entry. Add dummy value to make an index of 0 mean no mapping. + inverse := [][4]byte{{}} + mapping := map[[4]byte]int{[4]byte{}: 0} + + getWidthData(func(r rune, tag elem, alt rune) { + idx := 0 + if alt != 0 { + var buf [4]byte + buf[0] = byte(utf8.EncodeRune(buf[1:], alt)) + s := string(r) + buf[buf[0]] ^= s[len(s)-1] + var ok bool + if idx, ok = mapping[buf]; !ok { + idx = len(mapping) + if idx > math.MaxUint8 { + log.Fatalf("Index %d does not fit in a byte.", idx) + } + mapping[buf] = idx + inverse = append(inverse, buf) + } + } + t.Insert(r, uint64(tag|elem(idx))) + }) + + w := &bytes.Buffer{} + gen.WriteUnicodeVersion(w) + + sz, err := t.Gen(w) + if err != nil { + log.Fatal(err) + } + + sz += writeMappings(w, inverse) + + fmt.Fprintf(w, "// Total table size %d bytes (%dKiB)\n", sz, sz/1024) + + gen.WriteVersionedGoFile(*outputFile, "width", w.Bytes()) +} + +const inverseDataComment = ` +// inverseData contains 4-byte entries of the following format: +// <0 padding> +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// Ａ -> A (U+FF21 -> U+0041) +// Ｂ -> B (U+FF22 -> U+0042) +// ... +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// { 0x01, 0xE0, 0x00, 0x00 } +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// E0 ^ A1 = 41. +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// E0 ^ A2 = 42. +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8.` + +func writeMappings(w io.Writer, data [][4]byte) int { + fmt.Fprintln(w, inverseDataComment) + fmt.Fprintf(w, "var inverseData = [%d][4]byte{\n", len(data)) + for _, x := range data { + fmt.Fprintf(w, "{ 0x%02x, 0x%02x, 0x%02x, 0x%02x },\n", x[0], x[1], x[2], x[3]) + } + fmt.Fprintln(w, "}") + return len(data) * 4 +} + +func genTests() { + w := &bytes.Buffer{} + fmt.Fprintf(w, "\nvar mapRunes = map[rune]struct{r rune; e elem}{\n") + getWidthData(func(r rune, tag elem, alt rune) { + if alt != 0 { + fmt.Fprintf(w, "\t0x%X: {0x%X, 0x%X},\n", r, alt, tag) + } + }) + fmt.Fprintln(w, "}") + gen.WriteGoFile("runes_test.go", "width", w.Bytes()) +} diff --git a/vendor/golang.org/x/text/width/gen_common.go b/vendor/golang.org/x/text/width/gen_common.go new file mode 100644 index 0000000000..601e752684 --- /dev/null +++ b/vendor/golang.org/x/text/width/gen_common.go @@ -0,0 +1,96 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This code is shared between the main code generator and the test code. + +import ( + "flag" + "log" + "strconv" + "strings" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/ucd" +) + +var ( + outputFile = flag.String("out", "tables.go", "output file") +) + +var typeMap = map[string]elem{ + "A": tagAmbiguous, + "N": tagNeutral, + "Na": tagNarrow, + "W": tagWide, + "F": tagFullwidth, + "H": tagHalfwidth, +} + +// getWidthData calls f for every entry for which it is defined. +// +// f may be called multiple times for the same rune. The last call to f is the +// correct value. f is not called for all runes. The default tag type is +// Neutral. +func getWidthData(f func(r rune, tag elem, alt rune)) { + // Set the default values for Unified Ideographs. In line with Annex 11, + // we encode full ranges instead of the defined runes in Unified_Ideograph. + for _, b := range []struct{ lo, hi rune }{ + {0x4E00, 0x9FFF}, // the CJK Unified Ideographs block, + {0x3400, 0x4DBF}, // the CJK Unified Ideographs Externsion A block, + {0xF900, 0xFAFF}, // the CJK Compatibility Ideographs block, + {0x20000, 0x2FFFF}, // the Supplementary Ideographic Plane, + {0x30000, 0x3FFFF}, // the Tertiary Ideographic Plane, + } { + for r := b.lo; r <= b.hi; r++ { + f(r, tagWide, 0) + } + } + + inverse := map[rune]rune{} + maps := map[string]bool{ + "": true, + "": true, + } + + // We cannot reuse package norm's decomposition, as we need an unexpanded + // decomposition. We make use of the opportunity to verify that the + // decomposition type is as expected. + ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) { + r := p.Rune(0) + s := strings.SplitN(p.String(ucd.DecompMapping), " ", 2) + if !maps[s[0]] { + return + } + x, err := strconv.ParseUint(s[1], 16, 32) + if err != nil { + log.Fatalf("Error parsing rune %q", s[1]) + } + if inverse[r] != 0 || inverse[rune(x)] != 0 { + log.Fatalf("Circular dependency in mapping between %U and %U", r, x) + } + inverse[r] = rune(x) + inverse[rune(x)] = r + }) + + // ; + ucd.Parse(gen.OpenUCDFile("EastAsianWidth.txt"), func(p *ucd.Parser) { + tag, ok := typeMap[p.String(1)] + if !ok { + log.Fatalf("Unknown width type %q", p.String(1)) + } + r := p.Rune(0) + alt, ok := inverse[r] + if tag == tagFullwidth || tag == tagHalfwidth && r != wonSign { + tag |= tagNeedsFold + if !ok { + log.Fatalf("Narrow or wide rune %U has no decomposition", r) + } + } + f(r, tag, alt) + }) +} diff --git a/vendor/golang.org/x/text/width/gen_trieval.go b/vendor/golang.org/x/text/width/gen_trieval.go new file mode 100644 index 0000000000..c17334aa61 --- /dev/null +++ b/vendor/golang.org/x/text/width/gen_trieval.go @@ -0,0 +1,34 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// elem is an entry of the width trie. The high byte is used to encode the type +// of the rune. The low byte is used to store the index to a mapping entry in +// the inverseData array. +type elem uint16 + +const ( + tagNeutral elem = iota << typeShift + tagAmbiguous + tagWide + tagNarrow + tagFullwidth + tagHalfwidth +) + +const ( + numTypeBits = 3 + typeShift = 16 - numTypeBits + + // tagNeedsFold is true for all fullwidth and halfwidth runes except for + // the Won sign U+20A9. + tagNeedsFold = 0x1000 + + // The Korean Won sign is halfwidth, but SHOULD NOT be mapped to a wide + // variant. + wonSign rune = 0x20A9 +) diff --git a/vendor/golang.org/x/tools/go/gcexportdata/main.go b/vendor/golang.org/x/tools/go/gcexportdata/main.go new file mode 100644 index 0000000000..2713dce64a --- /dev/null +++ b/vendor/golang.org/x/tools/go/gcexportdata/main.go @@ -0,0 +1,99 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// The gcexportdata command is a diagnostic tool that displays the +// contents of gc export data files. +package main + +import ( + "flag" + "fmt" + "go/token" + "go/types" + "log" + "os" + + "golang.org/x/tools/go/gcexportdata" + "golang.org/x/tools/go/types/typeutil" +) + +var packageFlag = flag.String("package", "", "alternative package to print") + +func main() { + log.SetPrefix("gcexportdata: ") + log.SetFlags(0) + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "usage: gcexportdata [-package path] file.a") + } + flag.Parse() + if flag.NArg() != 1 { + flag.Usage() + os.Exit(2) + } + filename := flag.Args()[0] + + f, err := os.Open(filename) + if err != nil { + log.Fatal(err) + } + + r, err := gcexportdata.NewReader(f) + if err != nil { + log.Fatalf("%s: %s", filename, err) + } + + // Decode the package. + const primary = "" + imports := make(map[string]*types.Package) + fset := token.NewFileSet() + pkg, err := gcexportdata.Read(r, fset, imports, primary) + if err != nil { + log.Fatalf("%s: %s", filename, err) + } + + // Optionally select an indirectly mentioned package. + if *packageFlag != "" { + pkg = imports[*packageFlag] + if pkg == nil { + fmt.Fprintf(os.Stderr, "export data file %s does not mention %s; has:\n", + filename, *packageFlag) + for p := range imports { + if p != primary { + fmt.Fprintf(os.Stderr, "\t%s\n", p) + } + } + os.Exit(1) + } + } + + // Print all package-level declarations, including non-exported ones. + fmt.Printf("package %s\n", pkg.Name()) + for _, imp := range pkg.Imports() { + fmt.Printf("import %q\n", imp.Path()) + } + qual := func(p *types.Package) string { + if pkg == p { + return "" + } + return p.Name() + } + scope := pkg.Scope() + for _, name := range scope.Names() { + obj := scope.Lookup(name) + fmt.Printf("%s: %s\n", + fset.Position(obj.Pos()), + types.ObjectString(obj, qual)) + + // For types, print each method. + if _, ok := obj.(*types.TypeName); ok { + for _, method := range typeutil.IntuitiveMethodSet(obj.Type(), nil) { + fmt.Printf("%s: %s\n", + fset.Position(method.Obj().Pos()), + types.SelectionString(method, qual)) + } + } + } +} diff --git a/vendor/k8s.io/utils/pointer/OWNERS b/vendor/k8s.io/utils/pointer/OWNERS new file mode 100644 index 0000000000..0d6392752a --- /dev/null +++ b/vendor/k8s.io/utils/pointer/OWNERS @@ -0,0 +1,10 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +- apelisse +- stewart-yu +- thockin +reviewers: +- apelisse +- stewart-yu +- thockin diff --git a/vendor/k8s.io/utils/pointer/pointer.go b/vendor/k8s.io/utils/pointer/pointer.go new file mode 100644 index 0000000000..a11a540f46 --- /dev/null +++ b/vendor/k8s.io/utils/pointer/pointer.go @@ -0,0 +1,86 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pointer + +import ( + "fmt" + "reflect" +) + +// AllPtrFieldsNil tests whether all pointer fields in a struct are nil. This is useful when, +// for example, an API struct is handled by plugins which need to distinguish +// "no plugin accepted this spec" from "this spec is empty". +// +// This function is only valid for structs and pointers to structs. Any other +// type will cause a panic. Passing a typed nil pointer will return true. +func AllPtrFieldsNil(obj interface{}) bool { + v := reflect.ValueOf(obj) + if !v.IsValid() { + panic(fmt.Sprintf("reflect.ValueOf() produced a non-valid Value for %#v", obj)) + } + if v.Kind() == reflect.Ptr { + if v.IsNil() { + return true + } + v = v.Elem() + } + for i := 0; i < v.NumField(); i++ { + if v.Field(i).Kind() == reflect.Ptr && !v.Field(i).IsNil() { + return false + } + } + return true +} + +// Int32Ptr returns a pointer to an int32 +func Int32Ptr(i int32) *int32 { + return &i +} + +// Int64Ptr returns a pointer to an int64 +func Int64Ptr(i int64) *int64 { + return &i +} + +// Int32PtrDerefOr dereference the int32 ptr and returns it i not nil, +// else returns def. +func Int32PtrDerefOr(ptr *int32, def int32) int32 { + if ptr != nil { + return *ptr + } + return def +} + +// BoolPtr returns a pointer to a bool +func BoolPtr(b bool) *bool { + return &b +} + +// StringPtr returns a pointer to the passed string. +func StringPtr(s string) *string { + return &s +} + +// Float32Ptr returns a pointer to the passed float32. +func Float32Ptr(i float32) *float32 { + return &i +} + +// Float64Ptr returns a pointer to the passed float64. +func Float64Ptr(i float64) *float64 { + return &i +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c7c265972e..ef6d062461 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -14,39 +14,39 @@ github.com/appscode/jsonpatch # github.com/aws/aws-sdk-go v1.15.66 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/awserr -github.com/aws/aws-sdk-go/aws/awsutil -github.com/aws/aws-sdk-go/aws/client -github.com/aws/aws-sdk-go/aws/client/metadata -github.com/aws/aws-sdk-go/aws/corehandlers +github.com/aws/aws-sdk-go/service/ec2 +github.com/aws/aws-sdk-go/service/elb +github.com/aws/aws-sdk-go/service/elbv2 github.com/aws/aws-sdk-go/aws/credentials -github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds -github.com/aws/aws-sdk-go/aws/credentials/endpointcreds -github.com/aws/aws-sdk-go/aws/credentials/stscreds -github.com/aws/aws-sdk-go/aws/csm -github.com/aws/aws-sdk-go/aws/defaults -github.com/aws/aws-sdk-go/aws/ec2metadata -github.com/aws/aws-sdk-go/aws/endpoints github.com/aws/aws-sdk-go/aws/request github.com/aws/aws-sdk-go/aws/session -github.com/aws/aws-sdk-go/aws/signer/v4 -github.com/aws/aws-sdk-go/internal/ini +github.com/aws/aws-sdk-go/service/ec2/ec2iface +github.com/aws/aws-sdk-go/service/elb/elbiface +github.com/aws/aws-sdk-go/service/elbv2/elbv2iface +github.com/aws/aws-sdk-go/aws/endpoints github.com/aws/aws-sdk-go/internal/sdkio +github.com/aws/aws-sdk-go/aws/awsutil +github.com/aws/aws-sdk-go/aws/client +github.com/aws/aws-sdk-go/aws/client/metadata +github.com/aws/aws-sdk-go/aws/signer/v4 github.com/aws/aws-sdk-go/internal/sdkrand -github.com/aws/aws-sdk-go/internal/sdkuri -github.com/aws/aws-sdk-go/internal/shareddefaults github.com/aws/aws-sdk-go/private/protocol github.com/aws/aws-sdk-go/private/protocol/ec2query github.com/aws/aws-sdk-go/private/protocol/query -github.com/aws/aws-sdk-go/private/protocol/query/queryutil +github.com/aws/aws-sdk-go/internal/ini +github.com/aws/aws-sdk-go/internal/shareddefaults +github.com/aws/aws-sdk-go/aws/corehandlers +github.com/aws/aws-sdk-go/aws/credentials/stscreds +github.com/aws/aws-sdk-go/aws/csm +github.com/aws/aws-sdk-go/aws/defaults github.com/aws/aws-sdk-go/private/protocol/rest +github.com/aws/aws-sdk-go/private/protocol/query/queryutil github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil -github.com/aws/aws-sdk-go/service/ec2 -github.com/aws/aws-sdk-go/service/ec2/ec2iface -github.com/aws/aws-sdk-go/service/elb -github.com/aws/aws-sdk-go/service/elb/elbiface -github.com/aws/aws-sdk-go/service/elbv2 -github.com/aws/aws-sdk-go/service/elbv2/elbv2iface github.com/aws/aws-sdk-go/service/sts +github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds +github.com/aws/aws-sdk-go/aws/credentials/endpointcreds +github.com/aws/aws-sdk-go/aws/ec2metadata +github.com/aws/aws-sdk-go/internal/sdkuri # github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 github.com/beorn7/perks/quantile # github.com/blang/semver v3.5.1+incompatible @@ -63,8 +63,8 @@ github.com/fatih/color # github.com/ghodss/yaml v1.0.0 github.com/ghodss/yaml # github.com/go-log/log v0.0.0-20181211034820-a514cf01a3eb -github.com/go-log/log github.com/go-log/log/info +github.com/go-log/log # github.com/go-logr/logr v0.1.0 github.com/go-logr/logr # github.com/go-openapi/jsonpointer v0.19.2 @@ -85,13 +85,13 @@ github.com/golang/glog # github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff github.com/golang/groupcache/lru # github.com/golang/mock v1.1.1 -github.com/golang/mock/gomock github.com/golang/mock/mockgen +github.com/golang/mock/gomock github.com/golang/mock/mockgen/model # github.com/golang/protobuf v1.2.0 github.com/golang/protobuf/proto -github.com/golang/protobuf/ptypes github.com/golang/protobuf/ptypes/any +github.com/golang/protobuf/ptypes github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp # github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf @@ -122,9 +122,9 @@ github.com/json-iterator/go # github.com/konsorten/go-windows-terminal-sequences v1.0.1 github.com/konsorten/go-windows-terminal-sequences # github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 -github.com/mailru/easyjson/buffer github.com/mailru/easyjson/jlexer github.com/mailru/easyjson/jwriter +github.com/mailru/easyjson/buffer # github.com/mattn/go-colorable v0.1.2 github.com/mattn/go-colorable # github.com/mattn/go-isatty v0.0.8 @@ -139,68 +139,68 @@ github.com/modern-go/reflect2 github.com/onsi/ginkgo github.com/onsi/ginkgo/config github.com/onsi/ginkgo/internal/codelocation -github.com/onsi/ginkgo/internal/containernode github.com/onsi/ginkgo/internal/failer -github.com/onsi/ginkgo/internal/leafnodes github.com/onsi/ginkgo/internal/remote -github.com/onsi/ginkgo/internal/spec -github.com/onsi/ginkgo/internal/spec_iterator -github.com/onsi/ginkgo/internal/specrunner github.com/onsi/ginkgo/internal/suite github.com/onsi/ginkgo/internal/testingtproxy github.com/onsi/ginkgo/internal/writer github.com/onsi/ginkgo/reporters github.com/onsi/ginkgo/reporters/stenographer github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable -github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty github.com/onsi/ginkgo/types +github.com/onsi/ginkgo/internal/spec_iterator +github.com/onsi/ginkgo/internal/containernode +github.com/onsi/ginkgo/internal/leafnodes +github.com/onsi/ginkgo/internal/spec +github.com/onsi/ginkgo/internal/specrunner +github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty # github.com/onsi/gomega v1.5.0 github.com/onsi/gomega -github.com/onsi/gomega/format github.com/onsi/gomega/internal/assertion github.com/onsi/gomega/internal/asyncassertion -github.com/onsi/gomega/internal/oraclematcher github.com/onsi/gomega/internal/testingtsupport github.com/onsi/gomega/matchers +github.com/onsi/gomega/types +github.com/onsi/gomega/internal/oraclematcher +github.com/onsi/gomega/format github.com/onsi/gomega/matchers/support/goraph/bipartitegraph github.com/onsi/gomega/matchers/support/goraph/edge github.com/onsi/gomega/matchers/support/goraph/node github.com/onsi/gomega/matchers/support/goraph/util -github.com/onsi/gomega/types # github.com/openshift/cluster-api v0.0.0-20190917100308-655e2d6ccdd5 github.com/openshift/cluster-api/cmd/manager +github.com/openshift/cluster-api/pkg/apis/machine/v1beta1 github.com/openshift/cluster-api/pkg/apis -github.com/openshift/cluster-api/pkg/apis/cluster/common +github.com/openshift/cluster-api/pkg/controller/machine github.com/openshift/cluster-api/pkg/apis/cluster/v1alpha1 +github.com/openshift/cluster-api/pkg/controller/error +github.com/openshift/cluster-api/pkg/errors +github.com/openshift/cluster-api/pkg/controller github.com/openshift/cluster-api/pkg/apis/machine/common -github.com/openshift/cluster-api/pkg/apis/machine/v1beta1 +github.com/openshift/cluster-api/pkg/drain +github.com/openshift/cluster-api/pkg/util +github.com/openshift/cluster-api/pkg/apis/cluster/common github.com/openshift/cluster-api/pkg/client/clientset_generated/clientset -github.com/openshift/cluster-api/pkg/client/clientset_generated/clientset/scheme +github.com/openshift/cluster-api/pkg/controller/node +github.com/openshift/cluster-api/pkg/controller/machineset github.com/openshift/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1 github.com/openshift/cluster-api/pkg/client/clientset_generated/clientset/typed/machine/v1beta1 -github.com/openshift/cluster-api/pkg/controller -github.com/openshift/cluster-api/pkg/controller/error -github.com/openshift/cluster-api/pkg/controller/machine -github.com/openshift/cluster-api/pkg/controller/machineset -github.com/openshift/cluster-api/pkg/controller/node github.com/openshift/cluster-api/pkg/controller/noderefutil -github.com/openshift/cluster-api/pkg/drain -github.com/openshift/cluster-api/pkg/errors -github.com/openshift/cluster-api/pkg/util +github.com/openshift/cluster-api/pkg/client/clientset_generated/clientset/scheme # github.com/openshift/cluster-api-actuator-pkg v0.0.0-20190527090340-7628df78fb4c +github.com/openshift/cluster-api-actuator-pkg/pkg/types github.com/openshift/cluster-api-actuator-pkg/pkg/e2e/framework github.com/openshift/cluster-api-actuator-pkg/pkg/manifests -github.com/openshift/cluster-api-actuator-pkg/pkg/types # github.com/openshift/cluster-autoscaler-operator v0.0.0-20190521201101-62768a6ba480 github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1 github.com/openshift/cluster-autoscaler-operator/pkg/apis/autoscaling/v1beta1 # github.com/openshift/machine-api-operator v0.0.0-20190312153711-9650e16c9880 -github.com/openshift/machine-api-operator/pkg/apis/healthchecking github.com/openshift/machine-api-operator/pkg/apis/healthchecking/v1alpha1 github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned -github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned/scheme -github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned/typed/healthchecking/v1alpha1 github.com/openshift/machine-api-operator/pkg/util/conditions +github.com/openshift/machine-api-operator/pkg/apis/healthchecking +github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned/typed/healthchecking/v1alpha1 +github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned/scheme # github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 github.com/pborman/uuid # github.com/pkg/errors v0.8.1 @@ -208,21 +208,21 @@ github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 github.com/pmezard/go-difflib/difflib # github.com/prometheus/client_golang v0.9.2 +github.com/prometheus/client_golang/prometheus/promhttp github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/internal -github.com/prometheus/client_golang/prometheus/promhttp # github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 github.com/prometheus/client_model/go # github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 -github.com/prometheus/common/expfmt -github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/log +github.com/prometheus/common/expfmt github.com/prometheus/common/model +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg # github.com/prometheus/procfs v0.0.0-20190209105433-f8d8b3f739bd github.com/prometheus/procfs -github.com/prometheus/procfs/internal/util github.com/prometheus/procfs/nfs github.com/prometheus/procfs/xfs +github.com/prometheus/procfs/internal/util # github.com/sirupsen/logrus v1.2.0 github.com/sirupsen/logrus # github.com/spf13/cobra v0.0.3 @@ -232,24 +232,24 @@ github.com/spf13/pflag # github.com/stretchr/testify v1.3.0 github.com/stretchr/testify/assert # golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 +golang.org/x/crypto/ssh +golang.org/x/crypto/ssh/terminal golang.org/x/crypto/curve25519 golang.org/x/crypto/ed25519 -golang.org/x/crypto/ed25519/internal/edwards25519 golang.org/x/crypto/internal/chacha20 -golang.org/x/crypto/internal/subtle golang.org/x/crypto/poly1305 -golang.org/x/crypto/ssh -golang.org/x/crypto/ssh/terminal +golang.org/x/crypto/ed25519/internal/edwards25519 +golang.org/x/crypto/internal/subtle # golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 golang.org/x/net/context -golang.org/x/net/context/ctxhttp -golang.org/x/net/html -golang.org/x/net/html/atom -golang.org/x/net/html/charset -golang.org/x/net/http/httpguts golang.org/x/net/http2 +golang.org/x/net/http/httpguts golang.org/x/net/http2/hpack golang.org/x/net/idna +golang.org/x/net/html/charset +golang.org/x/net/context/ctxhttp +golang.org/x/net/html +golang.org/x/net/html/atom # golang.org/x/oauth2 v0.0.0-20181031022657-8527f56f7107 golang.org/x/oauth2 golang.org/x/oauth2/google @@ -257,54 +257,54 @@ golang.org/x/oauth2/internal golang.org/x/oauth2/jws golang.org/x/oauth2/jwt # golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f -golang.org/x/sys/cpu +golang.org/x/sys/windows/svc/eventlog golang.org/x/sys/unix golang.org/x/sys/windows +golang.org/x/sys/cpu golang.org/x/sys/windows/registry -golang.org/x/sys/windows/svc/eventlog # golang.org/x/text v0.3.2 +golang.org/x/text/secure/bidirule +golang.org/x/text/unicode/bidi +golang.org/x/text/unicode/norm golang.org/x/text/encoding golang.org/x/text/encoding/charmap golang.org/x/text/encoding/htmlindex -golang.org/x/text/encoding/internal +golang.org/x/text/transform golang.org/x/text/encoding/internal/identifier +golang.org/x/text/encoding/internal golang.org/x/text/encoding/japanese golang.org/x/text/encoding/korean golang.org/x/text/encoding/simplifiedchinese golang.org/x/text/encoding/traditionalchinese golang.org/x/text/encoding/unicode -golang.org/x/text/internal/language -golang.org/x/text/internal/language/compact -golang.org/x/text/internal/tag -golang.org/x/text/internal/utf8internal golang.org/x/text/language +golang.org/x/text/internal/utf8internal golang.org/x/text/runes -golang.org/x/text/secure/bidirule -golang.org/x/text/transform -golang.org/x/text/unicode/bidi -golang.org/x/text/unicode/norm +golang.org/x/text/internal/language +golang.org/x/text/internal/language/compact golang.org/x/text/width +golang.org/x/text/internal/tag # golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 golang.org/x/time/rate # golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 +golang.org/x/tools/go/packages golang.org/x/tools/go/gcexportdata -golang.org/x/tools/go/internal/gcimporter golang.org/x/tools/go/internal/packagesdriver -golang.org/x/tools/go/packages -golang.org/x/tools/internal/fastwalk golang.org/x/tools/internal/gopathwalk golang.org/x/tools/internal/semver +golang.org/x/tools/go/internal/gcimporter +golang.org/x/tools/internal/fastwalk # google.golang.org/appengine v1.2.0 google.golang.org/appengine +google.golang.org/appengine/urlfetch google.golang.org/appengine/internal google.golang.org/appengine/internal/app_identity +google.golang.org/appengine/internal/modules +google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/internal/base google.golang.org/appengine/internal/datastore google.golang.org/appengine/internal/log -google.golang.org/appengine/internal/modules google.golang.org/appengine/internal/remote_api -google.golang.org/appengine/internal/urlfetch -google.golang.org/appengine/urlfetch # gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/alecthomas/kingpin.v2 # gopkg.in/fsnotify.v1 v1.4.7 @@ -316,7 +316,7 @@ gopkg.in/tomb.v1 # gopkg.in/yaml.v2 v2.2.2 gopkg.in/yaml.v2 # k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b -k8s.io/api/admission/v1beta1 +k8s.io/api/core/v1 k8s.io/api/admissionregistration/v1beta1 k8s.io/api/apps/v1 k8s.io/api/apps/v1beta1 @@ -335,7 +335,6 @@ k8s.io/api/batch/v2alpha1 k8s.io/api/certificates/v1beta1 k8s.io/api/coordination/v1 k8s.io/api/coordination/v1beta1 -k8s.io/api/core/v1 k8s.io/api/events/v1beta1 k8s.io/api/extensions/v1beta1 k8s.io/api/networking/v1 @@ -353,64 +352,83 @@ k8s.io/api/settings/v1alpha1 k8s.io/api/storage/v1 k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 +k8s.io/api/admission/v1beta1 # k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8 -k8s.io/apiextensions-apiserver/pkg/apis/apiextensions -k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset -k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme +k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1 +k8s.io/apiextensions-apiserver/pkg/apis/apiextensions +k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme # k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d +k8s.io/apimachinery/pkg/runtime k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors -k8s.io/apimachinery/pkg/api/meta -k8s.io/apimachinery/pkg/api/resource -k8s.io/apimachinery/pkg/apis/meta/internalversion k8s.io/apimachinery/pkg/apis/meta/v1 -k8s.io/apimachinery/pkg/apis/meta/v1/unstructured +k8s.io/apimachinery/pkg/types +k8s.io/apimachinery/pkg/util/errors +k8s.io/apimachinery/pkg/runtime/schema +k8s.io/apimachinery/pkg/runtime/serializer +k8s.io/apimachinery/pkg/util/wait k8s.io/apimachinery/pkg/apis/meta/v1/validation -k8s.io/apimachinery/pkg/apis/meta/v1beta1 +k8s.io/apimachinery/pkg/labels +k8s.io/apimachinery/pkg/util/intstr +k8s.io/apimachinery/pkg/util/validation/field +k8s.io/apimachinery/pkg/api/resource k8s.io/apimachinery/pkg/conversion k8s.io/apimachinery/pkg/conversion/queryparams +k8s.io/apimachinery/pkg/util/json +k8s.io/apimachinery/pkg/util/naming +k8s.io/apimachinery/pkg/util/runtime +k8s.io/apimachinery/pkg/util/sets +k8s.io/apimachinery/pkg/util/clock +k8s.io/apimachinery/pkg/util/strategicpatch +k8s.io/apimachinery/pkg/watch +k8s.io/apimachinery/pkg/api/meta +k8s.io/apimachinery/pkg/util/net +k8s.io/apimachinery/pkg/util/yaml k8s.io/apimachinery/pkg/fields -k8s.io/apimachinery/pkg/labels -k8s.io/apimachinery/pkg/runtime -k8s.io/apimachinery/pkg/runtime/schema -k8s.io/apimachinery/pkg/runtime/serializer +k8s.io/apimachinery/pkg/selection +k8s.io/apimachinery/pkg/runtime/serializer/streaming +k8s.io/apimachinery/pkg/apis/meta/v1/unstructured k8s.io/apimachinery/pkg/runtime/serializer/json k8s.io/apimachinery/pkg/runtime/serializer/protobuf k8s.io/apimachinery/pkg/runtime/serializer/recognizer -k8s.io/apimachinery/pkg/runtime/serializer/streaming k8s.io/apimachinery/pkg/runtime/serializer/versioning -k8s.io/apimachinery/pkg/selection -k8s.io/apimachinery/pkg/types -k8s.io/apimachinery/pkg/util/cache -k8s.io/apimachinery/pkg/util/clock -k8s.io/apimachinery/pkg/util/diff -k8s.io/apimachinery/pkg/util/errors -k8s.io/apimachinery/pkg/util/framer -k8s.io/apimachinery/pkg/util/intstr -k8s.io/apimachinery/pkg/util/json -k8s.io/apimachinery/pkg/util/mergepatch -k8s.io/apimachinery/pkg/util/naming -k8s.io/apimachinery/pkg/util/net k8s.io/apimachinery/pkg/util/rand -k8s.io/apimachinery/pkg/util/runtime -k8s.io/apimachinery/pkg/util/sets -k8s.io/apimachinery/pkg/util/strategicpatch k8s.io/apimachinery/pkg/util/uuid k8s.io/apimachinery/pkg/util/validation -k8s.io/apimachinery/pkg/util/validation/field -k8s.io/apimachinery/pkg/util/wait -k8s.io/apimachinery/pkg/util/yaml -k8s.io/apimachinery/pkg/version -k8s.io/apimachinery/pkg/watch -k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/reflect +k8s.io/apimachinery/pkg/util/mergepatch +k8s.io/apimachinery/third_party/forked/golang/json +k8s.io/apimachinery/pkg/apis/meta/v1beta1 +k8s.io/apimachinery/pkg/version +k8s.io/apimachinery/pkg/util/framer +k8s.io/apimachinery/pkg/util/cache +k8s.io/apimachinery/pkg/util/diff +k8s.io/apimachinery/pkg/apis/meta/internalversion # k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible -k8s.io/client-go/discovery -k8s.io/client-go/dynamic -k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/scheme +k8s.io/client-go/tools/record +k8s.io/client-go/plugin/pkg/client/auth/gcp +k8s.io/client-go/rest +k8s.io/client-go/kubernetes +k8s.io/client-go/tools/record/util +k8s.io/client-go/tools/reference +k8s.io/client-go/util/flowcontrol +k8s.io/client-go/testing +k8s.io/client-go/util/jsonpath +k8s.io/client-go/tools/clientcmd +k8s.io/client-go/tools/leaderelection +k8s.io/client-go/tools/leaderelection/resourcelock +k8s.io/client-go/pkg/version +k8s.io/client-go/plugin/pkg/client/auth/exec +k8s.io/client-go/rest/watch +k8s.io/client-go/tools/clientcmd/api +k8s.io/client-go/tools/metrics +k8s.io/client-go/transport +k8s.io/client-go/util/cert +k8s.io/client-go/dynamic +k8s.io/client-go/discovery k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 k8s.io/client-go/kubernetes/typed/apps/v1 k8s.io/client-go/kubernetes/typed/apps/v1beta1 @@ -447,93 +465,76 @@ k8s.io/client-go/kubernetes/typed/settings/v1alpha1 k8s.io/client-go/kubernetes/typed/storage/v1 k8s.io/client-go/kubernetes/typed/storage/v1alpha1 k8s.io/client-go/kubernetes/typed/storage/v1beta1 -k8s.io/client-go/pkg/apis/clientauthentication -k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 -k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 -k8s.io/client-go/pkg/version -k8s.io/client-go/plugin/pkg/client/auth/exec -k8s.io/client-go/plugin/pkg/client/auth/gcp -k8s.io/client-go/rest -k8s.io/client-go/rest/watch +k8s.io/client-go/tools/cache k8s.io/client-go/restmapper -k8s.io/client-go/testing +k8s.io/client-go/util/workqueue k8s.io/client-go/third_party/forked/golang/template k8s.io/client-go/tools/auth -k8s.io/client-go/tools/cache -k8s.io/client-go/tools/clientcmd -k8s.io/client-go/tools/clientcmd/api k8s.io/client-go/tools/clientcmd/api/latest -k8s.io/client-go/tools/clientcmd/api/v1 -k8s.io/client-go/tools/leaderelection -k8s.io/client-go/tools/leaderelection/resourcelock -k8s.io/client-go/tools/metrics -k8s.io/client-go/tools/pager -k8s.io/client-go/tools/record -k8s.io/client-go/tools/record/util -k8s.io/client-go/tools/reference -k8s.io/client-go/transport -k8s.io/client-go/util/cert -k8s.io/client-go/util/connrotation -k8s.io/client-go/util/flowcontrol k8s.io/client-go/util/homedir -k8s.io/client-go/util/jsonpath +k8s.io/client-go/pkg/apis/clientauthentication +k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 +k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 +k8s.io/client-go/util/connrotation k8s.io/client-go/util/keyutil +k8s.io/client-go/tools/pager k8s.io/client-go/util/retry -k8s.io/client-go/util/workqueue +k8s.io/client-go/tools/clientcmd/api/v1 # k8s.io/klog v0.3.1 k8s.io/klog # k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058 -k8s.io/kube-openapi/pkg/common k8s.io/kube-openapi/pkg/util/proto +k8s.io/kube-openapi/pkg/common # k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 -k8s.io/utils/buffer +k8s.io/utils/pointer k8s.io/utils/integer +k8s.io/utils/buffer k8s.io/utils/trace # sigs.k8s.io/controller-runtime v0.0.0-20190520212815-96b67f231945 -sigs.k8s.io/controller-runtime/pkg/cache -sigs.k8s.io/controller-runtime/pkg/cache/internal +sigs.k8s.io/controller-runtime/pkg/client/fake +sigs.k8s.io/controller-runtime/pkg/client/config +sigs.k8s.io/controller-runtime/pkg/manager +sigs.k8s.io/controller-runtime/pkg/runtime/signals sigs.k8s.io/controller-runtime/pkg/client +sigs.k8s.io/controller-runtime/pkg/scheme sigs.k8s.io/controller-runtime/pkg/client/apiutil -sigs.k8s.io/controller-runtime/pkg/client/config -sigs.k8s.io/controller-runtime/pkg/client/fake +sigs.k8s.io/controller-runtime/pkg/internal/objectutil sigs.k8s.io/controller-runtime/pkg/controller -sigs.k8s.io/controller-runtime/pkg/conversion -sigs.k8s.io/controller-runtime/pkg/event sigs.k8s.io/controller-runtime/pkg/handler -sigs.k8s.io/controller-runtime/pkg/internal/controller -sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics +sigs.k8s.io/controller-runtime/pkg/reconcile +sigs.k8s.io/controller-runtime/pkg/source sigs.k8s.io/controller-runtime/pkg/internal/log -sigs.k8s.io/controller-runtime/pkg/internal/objectutil +sigs.k8s.io/controller-runtime/pkg/cache sigs.k8s.io/controller-runtime/pkg/internal/recorder sigs.k8s.io/controller-runtime/pkg/leaderelection -sigs.k8s.io/controller-runtime/pkg/log -sigs.k8s.io/controller-runtime/pkg/manager -sigs.k8s.io/controller-runtime/pkg/manager/signals sigs.k8s.io/controller-runtime/pkg/metrics -sigs.k8s.io/controller-runtime/pkg/predicate -sigs.k8s.io/controller-runtime/pkg/reconcile sigs.k8s.io/controller-runtime/pkg/recorder sigs.k8s.io/controller-runtime/pkg/runtime/inject -sigs.k8s.io/controller-runtime/pkg/runtime/signals -sigs.k8s.io/controller-runtime/pkg/scheme -sigs.k8s.io/controller-runtime/pkg/source -sigs.k8s.io/controller-runtime/pkg/source/internal sigs.k8s.io/controller-runtime/pkg/webhook -sigs.k8s.io/controller-runtime/pkg/webhook/admission sigs.k8s.io/controller-runtime/pkg/webhook/conversion +sigs.k8s.io/controller-runtime/pkg/manager/signals +sigs.k8s.io/controller-runtime/pkg/internal/controller +sigs.k8s.io/controller-runtime/pkg/predicate +sigs.k8s.io/controller-runtime/pkg/event +sigs.k8s.io/controller-runtime/pkg/source/internal +sigs.k8s.io/controller-runtime/pkg/log +sigs.k8s.io/controller-runtime/pkg/cache/internal +sigs.k8s.io/controller-runtime/pkg/webhook/admission sigs.k8s.io/controller-runtime/pkg/webhook/internal/certwatcher sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics +sigs.k8s.io/controller-runtime/pkg/conversion +sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics # sigs.k8s.io/controller-tools v0.2.0 sigs.k8s.io/controller-tools/cmd/controller-gen sigs.k8s.io/controller-tools/pkg/crd -sigs.k8s.io/controller-tools/pkg/crd/markers sigs.k8s.io/controller-tools/pkg/deepcopy sigs.k8s.io/controller-tools/pkg/genall sigs.k8s.io/controller-tools/pkg/genall/help sigs.k8s.io/controller-tools/pkg/genall/help/pretty -sigs.k8s.io/controller-tools/pkg/loader sigs.k8s.io/controller-tools/pkg/markers sigs.k8s.io/controller-tools/pkg/rbac sigs.k8s.io/controller-tools/pkg/webhook +sigs.k8s.io/controller-tools/pkg/crd/markers +sigs.k8s.io/controller-tools/pkg/loader # sigs.k8s.io/yaml v1.1.0 sigs.k8s.io/yaml