diff --git a/backend/core/models/domainlayer/code/repo.go b/backend/core/models/domainlayer/code/repo.go index 90469c1cb9f..f6ad9faf98b 100644 --- a/backend/core/models/domainlayer/code/repo.go +++ b/backend/core/models/domainlayer/code/repo.go @@ -19,7 +19,7 @@ package code import ( "github.com/apache/incubator-devlake/core/models/domainlayer" - plugin "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/plugin" "time" ) @@ -30,11 +30,11 @@ type Repo struct { Name string `json:"name"` Url string `json:"url"` Description string `json:"description"` - OwnerId string `json:"ownerId" gorm:"type:varchar(255)"` + OwnerId string `json:"owner_id" gorm:"type:varchar(255)"` Language string `json:"language" gorm:"type:varchar(255)"` - ForkedFrom string `json:"forkedFrom"` - CreatedDate *time.Time `json:"createdDate"` - UpdatedDate *time.Time `json:"updatedDate"` + ForkedFrom string `json:"forked_from"` + CreatedDate *time.Time `json:"created_date"` + UpdatedDate *time.Time `json:"updated_date"` Deleted bool `json:"deleted"` } diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go index 911bfb6baac..6788c905753 100644 --- a/backend/core/plugin/hub.go +++ b/backend/core/plugin/hub.go @@ -79,6 +79,8 @@ func InitPlugins(basicRes context.BasicRes) { if err != nil { panic(fmt.Errorf("failed to initialize plugin %v due to %w", pluginName, err)) } + } else { + basicRes.GetLogger().Info("plugin: %s doesn't implement 'PluginInit', it will be skipped.", pluginName) } } } diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py b/backend/python/plugins/azuredevops/azuredevops/main.py index a58d5359148..966aca974f5 100644 --- a/backend/python/plugins/azuredevops/azuredevops/main.py +++ b/backend/python/plugins/azuredevops/azuredevops/main.py @@ -4,7 +4,6 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software @@ -27,6 +26,7 @@ from pydevlake.domain_layer.devops import CicdScope from pydevlake.pipeline_tasks import gitextractor, refdiff from pydevlake.api import APIException +from pydevlake.extractor import autoextract _SUPPORTED_EXTERNAL_SOURCE_PROVIDERS = ['github', 'githubenterprise', 'bitbucket', 'git'] @@ -50,13 +50,14 @@ def domain_scopes(self, git_repo: GitRepository): yield Repo( name=git_repo.name, url=git_repo.url, - forked_from=git_repo.parent_repository_url + forked_from=git_repo.parent_repository_url, + updated_date=git_repo.updated_date, ) - yield CicdScope( name=git_repo.name, description=git_repo.name, - url=git_repo.url + url=git_repo.url, + updatedDate=git_repo.updated_date, ) def remote_scope_groups(self, connection) -> list[RemoteScopeGroup]: @@ -88,7 +89,7 @@ def remote_scopes(self, connection, group_id: str) -> list[GitRepository]: url = urlparse(raw_repo['remoteUrl']) url = url._replace(netloc=url.hostname) raw_repo['url'] = url.geturl() - repo = GitRepository(**raw_repo) + repo = autoextract(raw_repo, GitRepository) if not repo.default_branch: continue if "parentRepository" in raw_repo: diff --git a/backend/python/plugins/azuredevops/azuredevops/migrations.py b/backend/python/plugins/azuredevops/azuredevops/migrations.py index fd3b0d58efe..b6924e03740 100644 --- a/backend/python/plugins/azuredevops/azuredevops/migrations.py +++ b/backend/python/plugins/azuredevops/azuredevops/migrations.py @@ -192,3 +192,10 @@ def add_queue_time_field_in_tool_azuredevops_builds(b: MigrationScriptBuilder): table = '_tool_azuredevops_builds' b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time timestamptz', Dialect.POSTGRESQL) b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time datetime', Dialect.MYSQL) + + +@migration(20240223170000, name="add updated_date field in _tool_azuredevops_gitrepositories") +def add_updated_date_field_in_tool_azuredevops_gitrepositories(b: MigrationScriptBuilder): + table = "_tool_azuredevops_gitrepositories" + b.execute(f'ALTER TABLE {table} add COLUMN updated_date datetime(3) DEFAULT NULL', Dialect.MYSQL) + b.execute(f'ALTER TABLE {table} add COLUMN updated_date TIMESTAMPTZ DEFAULT NULL', Dialect.POSTGRESQL) diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py b/backend/python/plugins/azuredevops/azuredevops/models.py index cbddd9d1d1e..3a295aa7c45 100644 --- a/backend/python/plugins/azuredevops/azuredevops/models.py +++ b/backend/python/plugins/azuredevops/azuredevops/models.py @@ -45,8 +45,9 @@ class GitRepository(ToolScope, table=True): default_branch: Optional[str] project_id: str org_id: str - parent_repository_url: Optional[str] = Field(source='parentRepository/url') + parent_repository_url: Optional[str] = Field(source='/parentRepository/url') provider: Optional[str] + updated_date: datetime.datetime = Field(source='/project/lastUpdateTime') def is_external(self): return bool(self.provider) diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py index 6922d1b7ec5..bebb7bed872 100644 --- a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py +++ b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py @@ -37,7 +37,7 @@ def collect(self, state, context) -> Iterable[tuple[object, dict]]: def convert(self, pr: GitPullRequest, ctx): repo_id = ctx.scope.domain_id() - # If the PR is from a fork, we forge a new repo ID for the base repo but it doesn't correspond to a real repo + # If the PR is from a fork, we forge a new repo ID for the base repo, but it doesn't correspond to a real repo base_repo_id = domain_id(GitRepository, ctx.connection.id, pr.fork_repo_id) if pr.fork_repo_id is not None else repo_id # Use the same status values as GitHub plugin @@ -56,7 +56,7 @@ def convert(self, pr: GitPullRequest, ctx): original_status=pr.status.value, title=pr.title, description=pr.description, - url=pr.url, + url=f"{ctx.scope.url}/pullrequest/{pr.pull_request_id}", author_name=pr.created_by_name, author_id=pr.created_by_id, pull_request_key=pr.pull_request_id, diff --git a/backend/python/plugins/azuredevops/tests/streams_test.py b/backend/python/plugins/azuredevops/tests/streams_test.py index 97b963a7d08..110cb3280a3 100644 --- a/backend/python/plugins/azuredevops/tests/streams_test.py +++ b/backend/python/plugins/azuredevops/tests/streams_test.py @@ -287,7 +287,7 @@ def test_pull_requests_stream(context): original_status='active', title='ticket-2 PR', description='Updated main.java by ticket-2', - url='https://dev.azure.com/johndoe/7a3fd40e-2aed-4fac-bac9-511bf1a70206/_apis/git/repositories/0d50ba13-f9ad-49b0-9b21-d29eda50ca33/pullRequests/1', + url='https://github.com/johndoe/test-repo/pullrequest/1', author_name='John Doe', author_id='bc538feb-9fdd-6cf8-80e1-7c56950d0289', pull_request_key=1, diff --git a/backend/python/pydevlake/pydevlake/plugin.py b/backend/python/pydevlake/pydevlake/plugin.py index fc368261aea..e3303300ead 100644 --- a/backend/python/pydevlake/pydevlake/plugin.py +++ b/backend/python/pydevlake/pydevlake/plugin.py @@ -71,7 +71,7 @@ def scope_config_type(self) -> Type[ScopeConfig]: @abstractmethod def test_connection(self, connection: Connection) -> msg.TestConnectionResult: """ - Test if the the connection with the datasource can be established with the given connection. + Test if the connection with the datasource can be established with the given connection. Must raise an exception if the connection can't be established. """ pass