build(python): migrate pydevlake and plugins to Pydantic 2

.github/dependabot.yml

@@ -138,17 +138,11 @@ updates:
           - minor
           - patch
     ignore:
-      # pydevlake is written against Pydantic 1 and SQLModel 0.0.8. SQLModel is
-      # still pre-1.0, so SemVer treats even 0.0.8 -> 0.0.30 as a patch, yet that
-      # release pulls in SQLAlchemy 2 and Pydantic 2 and breaks
-      # `pydevlake/model.py` with "Passing primary_key is not supported when also
-      # passing a sa_column". Verified on a fork: the bump landed in the group
-      # and turned unit-test and test-e2e red.
-      # Drop both entries once pydevlake has been migrated to Pydantic 2.
+      # SQLModel is still pre-1.0, so SemVer treats every 0.0.x release as a
+      # patch, yet they change behavior: 0.0.48 rejects naive datetimes (which
+      # pydevlake stores) and pulls in SQLAlchemy 2.1. Bump it by hand and run
+      # the Python unit tests plus test-e2e (fakeplugin).
       - dependency-name: sqlmodel
-      - dependency-name: pydantic
-        update-types:
-          - version-update:semver-major
 
   # ------------------------------------------------------------- Docker ----
   #

backend/python/plugins/azuredevops/azuredevops/migrations.py

@@ -30,24 +30,24 @@
 def init_schemas(b: MigrationScriptBuilder):
     class AzureDevOpsConnection(Connection):
         token: SecretStr
-        organization: Optional[str]
+        organization: Optional[str] = None
 
     class AzureDevopsTransformationRule(ToolTable, Model):
         name: str = Field(default="default")
-        refdiff: Optional[RefDiffOptions]
-        deployment_pattern: Optional[re.Pattern]
-        production_pattern: Optional[re.Pattern]
+        refdiff: Optional[RefDiffOptions] = None
+        deployment_pattern: Optional[re.Pattern] = None
+        production_pattern: Optional[re.Pattern] = None
 
     class GitRepository(ToolModel):
         id: str = Field(primary_key=True)
         name: str
         url: str
-        remote_url: Optional[str]
-        default_branch: Optional[str]
+        remote_url: Optional[str] = None
+        default_branch: Optional[str] = None
         project_id: str
         org_id: str
-        parent_repository_url: Optional[str]
-        provider: Optional[str]
+        parent_repository_url: Optional[str] = None
+        provider: Optional[str] = None
 
     class GitPullRequest(ToolModel):
         class PRStatus(Enum):
@@ -56,21 +56,21 @@ class PRStatus(Enum):
             Completed = "completed"
 
         pull_request_id: int = Field(primary_key=True)
-        description: Optional[str]
+        description: Optional[str] = None
         status: PRStatus
         created_by_id: str
         created_by_name: str
         creation_date: datetime.datetime
-        closed_date: Optional[datetime.datetime]
+        closed_date: Optional[datetime.datetime] = None
         source_commit_sha: str
         target_commit_sha: str
-        merge_commit_sha: Optional[str]
-        url: Optional[str]
-        type: Optional[str]
-        title: Optional[str]
-        target_ref_name: Optional[str]
-        source_ref_name: Optional[str]
-        fork_repo_id: Optional[str]
+        merge_commit_sha: Optional[str] = None
+        url: Optional[str] = None
+        type: Optional[str] = None
+        title: Optional[str] = None
+        target_ref_name: Optional[str] = None
+        source_ref_name: Optional[str] = None
+        fork_repo_id: Optional[str] = None
 
     class GitPullRequestCommit(ToolModel):
         commit_id: str = Field(primary_key=True)
@@ -96,10 +96,10 @@ class BuildResult(Enum):
 
         id: int = Field(primary_key=True, auto_increment=False)
         name: str
-        start_time: Optional[datetime.datetime]
-        finish_time: Optional[datetime.datetime]
+        start_time: Optional[datetime.datetime] = None
+        finish_time: Optional[datetime.datetime] = None
         status: BuildStatus
-        result: Optional[BuildResult]
+        result: Optional[BuildResult] = None
         source_branch: str
         source_version: str
 
@@ -120,10 +120,10 @@ class JobResult(Enum):
         id: str = Field(primary_key=True)
         build_id: str
         name: str
-        startTime: Optional[datetime.datetime]
-        finishTime: Optional[datetime.datetime]
+        startTime: Optional[datetime.datetime] = None
+        finishTime: Optional[datetime.datetime] = None
         state: JobState
-        result: Optional[JobResult]
+        result: Optional[JobResult] = None
 
     b.create_tables(
         AzureDevOpsConnection,

backend/python/plugins/azuredevops/azuredevops/models.py

@@ -30,26 +30,26 @@
 
 class AzureDevOpsConnection(Connection):
     token: SecretStr
-    organization: Optional[str]
+    organization: Optional[str] = None
 
 
 class GitRepositoryConfig(ScopeConfig):
-    refdiff: Optional[RefDiffOptions]
-    deployment_pattern: Optional[re.Pattern]
-    production_pattern: Optional[re.Pattern]
+    refdiff: Optional[RefDiffOptions] = None
+    deployment_pattern: Optional[re.Pattern] = None
+    production_pattern: Optional[re.Pattern] = None
     # Optional pattern with capture group to extract environment name from job/stage names
     # Example: r'(?:deploy|predeploy)[_-](.+?)(?:[_-](?:helm|terraform))?$' extracts 'xxxx-prod' from 'deploy_xxxx-prod_helm'
-    environment_pattern: Optional[re.Pattern]
+    environment_pattern: Optional[re.Pattern] = None
 
 
 class GitRepository(ToolScope, table=True):
     url: str
-    remote_url: Optional[str]
-    default_branch: Optional[str]
+    remote_url: Optional[str] = None
+    default_branch: Optional[str] = None
     project_id: str
     org_id: str
-    parent_repository_url: Optional[str] = Field(source='/parentRepository/url')
-    provider: Optional[str]
+    parent_repository_url: Optional[str] = Field(default=None, source='/parentRepository/url')
+    provider: Optional[str] = None
     updated_date: datetime.datetime = Field(source='/project/lastUpdateTime')
 
     def is_external(self):
@@ -63,28 +63,28 @@ class PRStatus(Enum):
         Completed = "completed"
 
     pull_request_id: int = Field(primary_key=True)
-    description: Optional[str]
+    description: Optional[str] = None
     status: PRStatus
     created_by_id: str = Field(source='/createdBy/id')
     created_by_name: str = Field(source='/createdBy/displayName')
     creation_date: datetime.datetime
-    closed_date: Optional[datetime.datetime]
+    closed_date: Optional[datetime.datetime] = None
     source_commit_sha: str = Field(source='/lastMergeSourceCommit/commitId')
     target_commit_sha: str = Field(source='/lastMergeTargetCommit/commitId')
-    merge_commit_sha: Optional[str] = Field(source='/lastMergeCommit/commitId')
-    url: Optional[str]
-    type: Optional[str] = Field(source='/labels/0/name')  # TODO: Add regex to scope config
-    title: Optional[str]
-    target_ref_name: Optional[str]
-    source_ref_name: Optional[str]
-    fork_repo_id: Optional[str] = Field(source='/forkSource/repository/id')
+    merge_commit_sha: Optional[str] = Field(default=None, source='/lastMergeCommit/commitId')
+    url: Optional[str] = None
+    type: Optional[str] = Field(default=None, source='/labels/0/name')  # TODO: Add regex to scope config
+    title: Optional[str] = None
+    target_ref_name: Optional[str] = None
+    source_ref_name: Optional[str] = None
+    fork_repo_id: Optional[str] = Field(default=None, source='/forkSource/repository/id')
 
 
 class GitPullRequestCommit(ToolModel, table=True):
     commit_id: str = Field(primary_key=True)
     pull_request_id: str
     author_name: str = Field(source='/author/name')
-    author_email: Optional[str] = Field(source='/author/email')
+    author_email: Optional[str] = Field(default=None, source='/author/email')
     author_date: datetime.datetime = Field(source='/author/date')
 
 
@@ -111,15 +111,15 @@ def __str__(self) -> str:
 
     id: int = Field(primary_key=True)
     name: str = Field(source='/definition/name')
-    queue_time: Optional[datetime.datetime] = Field(source='/queueTime')
-    start_time: Optional[datetime.datetime]
-    finish_time: Optional[datetime.datetime]
+    queue_time: Optional[datetime.datetime] = Field(default=None, source='/queueTime')
+    start_time: Optional[datetime.datetime] = None
+    finish_time: Optional[datetime.datetime] = None
     status: BuildStatus
-    result: Optional[BuildResult]
+    result: Optional[BuildResult] = None
     source_branch: str
     source_version: str
-    display_title: Optional[str] = Field(source='/triggerInfo/ci.message')
-    url: Optional[str] = Field(source='/_links/web/href')
+    display_title: Optional[str] = Field(default=None, source='/triggerInfo/ci.message')
+    url: Optional[str] = Field(default=None, source='/_links/web/href')
 
 
 class Job(ToolModel, table=True):
@@ -145,10 +145,10 @@ def __str__(self) -> str:
     id: str = Field(primary_key=True)
     build_id: str = Field(primary_key=True)
     name: str
-    start_time: Optional[datetime.datetime]
-    finish_time: Optional[datetime.datetime]
+    start_time: Optional[datetime.datetime] = None
+    finish_time: Optional[datetime.datetime] = None
     state: JobState
-    result: Optional[JobResult]
-    identifier: Optional[str]
-    type: Optional[str]
-    parent_id: Optional[str] = Field(source='/parentId')
+    result: Optional[JobResult] = None
+    identifier: Optional[str] = None
+    type: Optional[str] = None
+    parent_id: Optional[str] = Field(default=None, source='/parentId')