cherry pick #7001 fix(azuredevops): fix some fields' value to v0.21 (#…

…7003)

* fix(azuredevops): fix some fields' value

* fix(azuredevops): fix updated_date in cicd_scopes

* fix(azuredevops): fix test

* fix(azuredevops): fix e2e test errors

backend/core/models/domainlayer/code/repo.go

@@ -19,7 +19,7 @@ package code
 
 import (
 	"github.com/apache/incubator-devlake/core/models/domainlayer"
-	plugin "github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/plugin"
 	"time"
 )
 
@@ -30,11 +30,11 @@ type Repo struct {
 	Name        string     `json:"name"`
 	Url         string     `json:"url"`
 	Description string     `json:"description"`
-	OwnerId     string     `json:"ownerId" gorm:"type:varchar(255)"`
+	OwnerId     string     `json:"owner_id" gorm:"type:varchar(255)"`
 	Language    string     `json:"language" gorm:"type:varchar(255)"`
-	ForkedFrom  string     `json:"forkedFrom"`
-	CreatedDate *time.Time `json:"createdDate"`
-	UpdatedDate *time.Time `json:"updatedDate"`
+	ForkedFrom  string     `json:"forked_from"`
+	CreatedDate *time.Time `json:"created_date"`
+	UpdatedDate *time.Time `json:"updated_date"`
 	Deleted     bool       `json:"deleted"`
 }
 

backend/core/plugin/hub.go

@@ -79,6 +79,8 @@ func InitPlugins(basicRes context.BasicRes) {
 			if err != nil {
 				panic(fmt.Errorf("failed to initialize plugin %v due to %w", pluginName, err))
 			}
+		} else {
+			basicRes.GetLogger().Info("plugin: %s doesn't implement 'PluginInit', it will be skipped.", pluginName)
 		}
 	}
 }

backend/python/plugins/azuredevops/azuredevops/main.py

@@ -4,7 +4,6 @@
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
-
 #     http://www.apache.org/licenses/LICENSE-2.0
 
 # Unless required by applicable law or agreed to in writing, software
@@ -27,6 +26,7 @@
 from pydevlake.domain_layer.devops import CicdScope
 from pydevlake.pipeline_tasks import gitextractor, refdiff
 from pydevlake.api import APIException
+from pydevlake.extractor import autoextract
 
 
 _SUPPORTED_EXTERNAL_SOURCE_PROVIDERS = ['github', 'githubenterprise', 'bitbucket', 'git']
@@ -50,13 +50,14 @@ def domain_scopes(self, git_repo: GitRepository):
         yield Repo(
             name=git_repo.name,
             url=git_repo.url,
-            forked_from=git_repo.parent_repository_url
+            forked_from=git_repo.parent_repository_url,
+            updated_date=git_repo.updated_date,
         )
-
         yield CicdScope(
             name=git_repo.name,
             description=git_repo.name,
-            url=git_repo.url
+            url=git_repo.url,
+            updatedDate=git_repo.updated_date,
         )
 
     def remote_scope_groups(self, connection) -> list[RemoteScopeGroup]:
@@ -88,7 +89,7 @@ def remote_scopes(self, connection, group_id: str) -> list[GitRepository]:
             url = urlparse(raw_repo['remoteUrl'])
             url = url._replace(netloc=url.hostname)
             raw_repo['url'] = url.geturl()
-            repo = GitRepository(**raw_repo)
+            repo = autoextract(raw_repo, GitRepository)
             if not repo.default_branch:
                 continue
             if "parentRepository" in raw_repo:

backend/python/plugins/azuredevops/azuredevops/migrations.py

@@ -192,3 +192,10 @@ def add_queue_time_field_in_tool_azuredevops_builds(b: MigrationScriptBuilder):
     table = '_tool_azuredevops_builds'
     b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time timestamptz', Dialect.POSTGRESQL)
     b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time datetime', Dialect.MYSQL)
+
+
+@migration(20240223170000, name="add updated_date field in _tool_azuredevops_gitrepositories")
+def add_updated_date_field_in_tool_azuredevops_gitrepositories(b: MigrationScriptBuilder):
+    table = "_tool_azuredevops_gitrepositories"
+    b.execute(f'ALTER TABLE {table} add COLUMN updated_date datetime(3) DEFAULT NULL', Dialect.MYSQL)
+    b.execute(f'ALTER TABLE {table} add COLUMN updated_date TIMESTAMPTZ DEFAULT NULL', Dialect.POSTGRESQL)

backend/python/plugins/azuredevops/azuredevops/models.py

@@ -45,8 +45,9 @@ class GitRepository(ToolScope, table=True):
     default_branch: Optional[str]
     project_id: str
     org_id: str
-    parent_repository_url: Optional[str] = Field(source='parentRepository/url')
+    parent_repository_url: Optional[str] = Field(source='/parentRepository/url')
     provider: Optional[str]
+    updated_date: datetime.datetime = Field(source='/project/lastUpdateTime')
 
     def is_external(self):
         return bool(self.provider)

backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py

@@ -37,7 +37,7 @@ def collect(self, state, context) -> Iterable[tuple[object, dict]]:
 
     def convert(self, pr: GitPullRequest, ctx):
         repo_id = ctx.scope.domain_id()
-        # If the PR is from a fork, we forge a new repo ID for the base repo but it doesn't correspond to a real repo
+        # If the PR is from a fork, we forge a new repo ID for the base repo, but it doesn't correspond to a real repo
         base_repo_id = domain_id(GitRepository, ctx.connection.id, pr.fork_repo_id) if pr.fork_repo_id is not None else repo_id
 
         # Use the same status values as GitHub plugin
@@ -56,7 +56,7 @@ def convert(self, pr: GitPullRequest, ctx):
             original_status=pr.status.value,
             title=pr.title,
             description=pr.description,
-            url=pr.url,
+            url=f"{ctx.scope.url}/pullrequest/{pr.pull_request_id}",
             author_name=pr.created_by_name,
             author_id=pr.created_by_id,
             pull_request_key=pr.pull_request_id,

backend/python/plugins/azuredevops/tests/streams_test.py

@@ -287,7 +287,7 @@ def test_pull_requests_stream(context):
         original_status='active',
         title='ticket-2 PR',
         description='Updated main.java by ticket-2',
-        url='https://dev.azure.com/johndoe/7a3fd40e-2aed-4fac-bac9-511bf1a70206/_apis/git/repositories/0d50ba13-f9ad-49b0-9b21-d29eda50ca33/pullRequests/1',
+        url='https://github.com/johndoe/test-repo/pullrequest/1',
         author_name='John Doe',
         author_id='bc538feb-9fdd-6cf8-80e1-7c56950d0289',
         pull_request_key=1,

backend/python/pydevlake/pydevlake/plugin.py

@@ -71,7 +71,7 @@ def scope_config_type(self) -> Type[ScopeConfig]:
     @abstractmethod
     def test_connection(self, connection: Connection) -> msg.TestConnectionResult:
         """
-        Test if the the connection with the datasource can be established with the given connection.
+        Test if the connection with the datasource can be established with the given connection.
         Must raise an exception if the connection can't be established.
         """
         pass