This repository has been archived by the owner on Mar 8, 2020. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: lwsanty <[email protected]>
- Loading branch information
Showing
6 changed files
with
169 additions
and
37 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| python: localhost:9432 | ||
| go: localhost:9432 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| package bblfsh | ||
|
|
||
| import ( | ||
| "context" | ||
| "io/ioutil" | ||
|
|
||
| "google.golang.org/grpc" | ||
| "gopkg.in/src-d/go-errors.v1" | ||
| "gopkg.in/yaml.v2" | ||
| ) | ||
|
|
||
| /* | ||
| MetaClient could be useful when dealing with language-specific parsings on the large scale | ||
| Scenario: we need to parse a ton of go and python files inside the k8s environment, to save time we need to perform this | ||
| parses on the large scale of go- and python-driver containers/instances etc. | ||
| Solution: | ||
| - run two separate Deployments of go- and python-driver container pods | ||
| - provide Horizontal Autoscalers for both of Deployments | ||
| - provide Services with LoadBalancer type | ||
| - during MetaClient initialization provide config with endpoints to Services, this will create two language-oriented clients | ||
| that will be responsible for sending parse request only to a dedicated Service, that will load-balance this request | ||
| between underlying language driver pods | ||
| */ | ||
|
|
||
| // ErrLangNotSupported is the error that must be returned if requested language driver is not supported/available | ||
| var ErrLangNotSupported = errors.NewKind("language %q is not supported in current configuration") | ||
|
|
||
| // MetaClient represents an entity that contains: | ||
| // 1) MetaClientConfig configuration | ||
| // 2) LangClients - map of already initialized clients | ||
| type MetaClient struct { | ||
| clients LangClients | ||
| config MetaClientConfig | ||
| } | ||
|
|
||
| // LangClients is a map, where key represents language name, value - corresponding client | ||
| type LangClients map[string]*Client | ||
|
|
||
| // MetaClientConfig is a map, where key represents language name, value - corresponding driver's API endpoint | ||
| type MetaClientConfig map[string]string | ||
|
|
||
| // NewMetaClient is a MetaClient constructor, used to parse and save MetaClientConfig from a given filepath | ||
| func NewMetaClient(configPath string) (*MetaClient, error) { | ||
| data, err := ioutil.ReadFile(configPath) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| config := make(MetaClientConfig) | ||
| if err := yaml.Unmarshal(data, config); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &MetaClient{ | ||
| clients: make(LangClients), | ||
| config: config, | ||
| }, nil | ||
| } | ||
|
|
||
| // GetClientContext returns corresponding client for requested language or creates the new one if it's not initialized | ||
| func (mc *MetaClient) GetClientContext( | ||
| ctx context.Context, | ||
| language string, | ||
| options ...grpc.DialOption) (*Client, error) { | ||
| if c, ok := mc.clients[language]; ok { | ||
| return c, nil | ||
| } | ||
| if endpoint, ok := mc.config[language]; ok { | ||
| c, err := NewClientContext(ctx, endpoint, options...) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return c, nil | ||
| } | ||
|
|
||
| return nil, ErrLangNotSupported.New(language) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| package bblfsh | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| const timeout = 5 * time.Second | ||
|
|
||
| func TestMetaClient(t *testing.T) { | ||
| mc, err := NewMetaClient("_testdata/endpoints.yml") | ||
| require.NoError(t, err) | ||
|
|
||
| for _, mct := range metaClientTests { | ||
| mct := mct | ||
| t.Run(mct.name, func(t *testing.T) { | ||
| mct.test(t, mc) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| var metaClientTests = []struct { | ||
| name string | ||
| test func(t *testing.T, mc *MetaClient) | ||
| }{ | ||
| {name: "GetClientContextAndParse", test: testGetClientContextAndParse}, | ||
| {name: "GetClientContextTwiceAndParse", test: testGetClientContextTwiceAndParse}, | ||
| {name: "GetNotSupportedClient", test: testGetNotSupportedClient}, | ||
| } | ||
|
|
||
| func testGetClientContextAndParse(t *testing.T, mc *MetaClient) { | ||
| ctx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| defer cancel() | ||
|
|
||
| getClientContextAndParse(t, ctx, mc, "python", "import foo") | ||
| getClientContextAndParse(t, ctx, mc, "go", "package main") | ||
| } | ||
|
|
||
| func testGetClientContextTwiceAndParse(t *testing.T, mc *MetaClient) { | ||
| ctx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| defer cancel() | ||
|
|
||
| getClientContextAndParse(t, ctx, mc, "python", "import foo") | ||
| getClientContextAndParse(t, ctx, mc, "python", "import foo") | ||
| } | ||
|
|
||
| func testGetNotSupportedClient(t *testing.T, mc *MetaClient) { | ||
| ctx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| defer cancel() | ||
|
|
||
| _, err := mc.GetClientContext(ctx, "lang") | ||
| if err == context.DeadlineExceeded { | ||
| t.Skip("bblfshd is not running") | ||
| } | ||
| require.True(t, ErrLangNotSupported.Is(err)) | ||
| } | ||
|
|
||
| func getClientContextAndParse( | ||
| t *testing.T, | ||
| ctx context.Context, | ||
| mc *MetaClient, | ||
| language, | ||
| content string) { | ||
| c, err := mc.GetClientContext(ctx, language) | ||
| if err == context.DeadlineExceeded { | ||
| t.Skip("bblfshd is not running") | ||
| } | ||
| require.NoError(t, err) | ||
|
|
||
| res, err := c.NewParseRequest().Mode(Native).Language(language).Content(content).Do() | ||
| require.NoError(t, err) | ||
|
|
||
| require.Len(t, res.Errors, 0) | ||
| require.NotNil(t, res) | ||
| } |