From 6b07e0e8d66e9c0ab7f43f817109ffd94960006f Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Aug 2025 11:25:36 +0800 Subject: [PATCH 1/3] feat: add expiration for OAuth credentials in datasource provider --- .../console/datasets/rag_pipeline/datasource_auth.py | 2 ++ api/models/oauth.py | 1 + api/services/datasource_provider_service.py | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/api/controllers/console/datasets/rag_pipeline/datasource_auth.py b/api/controllers/console/datasets/rag_pipeline/datasource_auth.py index d67af182cd..192d7ffdf6 100644 --- a/api/controllers/console/datasets/rag_pipeline/datasource_auth.py +++ b/api/controllers/console/datasets/rag_pipeline/datasource_auth.py @@ -110,6 +110,7 @@ class DatasourceOAuthCallback(Resource): provider_id=datasource_provider_id, avatar_url=oauth_response.metadata.get("avatar_url") or None, name=oauth_response.metadata.get("name") or None, + expire_at=oauth_response.expires_at, credentials=dict(oauth_response.credentials), credential_id=context.get("credential_id"), ) @@ -119,6 +120,7 @@ class DatasourceOAuthCallback(Resource): provider_id=datasource_provider_id, avatar_url=oauth_response.metadata.get("avatar_url") or None, name=oauth_response.metadata.get("name") or None, + expire_at=oauth_response.expires_at, credentials=dict(oauth_response.credentials), ) return redirect(f"{dify_config.CONSOLE_WEB_URL}/oauth-callback") diff --git a/api/models/oauth.py b/api/models/oauth.py index 8e661051a7..4848b86af2 100644 --- a/api/models/oauth.py +++ b/api/models/oauth.py @@ -37,6 +37,7 @@ class DatasourceProvider(Base): encrypted_credentials: Mapped[dict] = db.Column(JSONB, nullable=False) avatar_url: Mapped[str] = db.Column(db.String(255), nullable=True, default="default") is_default: Mapped[bool] = db.Column(db.Boolean, nullable=False, server_default=db.text("false")) + expires_at: Mapped[int] = db.Column(db.Integer, nullable=False, default=-1) created_at: Mapped[datetime] = db.Column(db.DateTime, nullable=False, default=datetime.now) updated_at: Mapped[datetime] = db.Column(db.DateTime, nullable=False, default=datetime.now) diff --git a/api/services/datasource_provider_service.py b/api/services/datasource_provider_service.py index a0b61c758e..ef29654a35 100644 --- a/api/services/datasource_provider_service.py +++ b/api/services/datasource_provider_service.py @@ -383,6 +383,7 @@ class DatasourceProviderService: tenant_id: str, provider_id: DatasourceProviderID, avatar_url: str | None, + expire_at: int, credentials: dict, credential_id: str, ) -> None: @@ -433,6 +434,7 @@ class DatasourceProviderService: if key in provider_credential_secret_variables: credentials[key] = encrypter.encrypt_token(tenant_id, value) + target_provider.expires_at = expire_at target_provider.encrypted_credentials = credentials target_provider.avatar_url = avatar_url or target_provider.avatar_url session.commit() @@ -443,6 +445,7 @@ class DatasourceProviderService: tenant_id: str, provider_id: DatasourceProviderID, avatar_url: str | None, + expire_at: int, credentials: dict, ) -> None: """ @@ -500,6 +503,7 @@ class DatasourceProviderService: auth_type=credential_type.value, encrypted_credentials=credentials, avatar_url=avatar_url or "default", + expires_at=expire_at, ) session.add(datasource_provider) session.commit() From ada0875ac4b9e4055d8c823eab96780ac65b14fa Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Aug 2025 11:40:51 +0800 Subject: [PATCH 2/3] fix: update default value for expires_at in DatasourceProvider model --- ..._1138-17d4db47800c_add_pipeline_info_16.py | 33 +++++++++++++++++++ api/models/oauth.py | 2 +- web/.vscode/launch.json | 15 +++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 api/migrations/versions/2025_08_11_1138-17d4db47800c_add_pipeline_info_16.py create mode 100644 web/.vscode/launch.json diff --git a/api/migrations/versions/2025_08_11_1138-17d4db47800c_add_pipeline_info_16.py b/api/migrations/versions/2025_08_11_1138-17d4db47800c_add_pipeline_info_16.py new file mode 100644 index 0000000000..6b056be4e9 --- /dev/null +++ b/api/migrations/versions/2025_08_11_1138-17d4db47800c_add_pipeline_info_16.py @@ -0,0 +1,33 @@ +"""datasource_oauth_refresh + +Revision ID: 17d4db47800c +Revises: 223c3f882c69 +Create Date: 2025-08-11 11:38:03.662874 + +""" +from alembic import op +import models as models +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '17d4db47800c' +down_revision = '223c3f882c69' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('datasource_providers', schema=None) as batch_op: + batch_op.add_column(sa.Column('expires_at', sa.Integer(), nullable=False, server_default='-1')) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('datasource_providers', schema=None) as batch_op: + batch_op.drop_column('expires_at') + + # ### end Alembic commands ### diff --git a/api/models/oauth.py b/api/models/oauth.py index 4848b86af2..23b204cf07 100644 --- a/api/models/oauth.py +++ b/api/models/oauth.py @@ -37,7 +37,7 @@ class DatasourceProvider(Base): encrypted_credentials: Mapped[dict] = db.Column(JSONB, nullable=False) avatar_url: Mapped[str] = db.Column(db.String(255), nullable=True, default="default") is_default: Mapped[bool] = db.Column(db.Boolean, nullable=False, server_default=db.text("false")) - expires_at: Mapped[int] = db.Column(db.Integer, nullable=False, default=-1) + expires_at: Mapped[int] = db.Column(db.Integer, nullable=False, server_default='-1') created_at: Mapped[datetime] = db.Column(db.DateTime, nullable=False, default=datetime.now) updated_at: Mapped[datetime] = db.Column(db.DateTime, nullable=False, default=datetime.now) diff --git a/web/.vscode/launch.json b/web/.vscode/launch.json new file mode 100644 index 0000000000..f6b35a0b63 --- /dev/null +++ b/web/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "chrome", + "request": "launch", + "name": "Launch Chrome against localhost", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}" + } + ] +} \ No newline at end of file From 087a4fbd144dfd2b07ce149b4e03ab97f8b1c68e Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Mon, 11 Aug 2025 18:26:41 +0800 Subject: [PATCH 3/3] add credential id --- api/core/datasource/entities/datasource_entities.py | 2 +- api/core/workflow/nodes/datasource/datasource_node.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/datasource/entities/datasource_entities.py b/api/core/datasource/entities/datasource_entities.py index 9ce580912f..d55e1392c3 100644 --- a/api/core/datasource/entities/datasource_entities.py +++ b/api/core/datasource/entities/datasource_entities.py @@ -359,5 +359,5 @@ class OnlineDriveDownloadFileRequest(BaseModel): Get online driver file """ - key: str = Field(..., description="The name of the file") + id: str = Field(..., description="The id of the file") bucket: Optional[str] = Field(None, description="The name of the bucket") diff --git a/api/core/workflow/nodes/datasource/datasource_node.py b/api/core/workflow/nodes/datasource/datasource_node.py index 201d062ade..d4f1c4f392 100644 --- a/api/core/workflow/nodes/datasource/datasource_node.py +++ b/api/core/workflow/nodes/datasource/datasource_node.py @@ -165,7 +165,7 @@ class DatasourceNode(BaseNode): datasource_runtime.online_drive_download_file( user_id=self.user_id, request=OnlineDriveDownloadFileRequest( - key=datasource_info.get("key"), + id=datasource_info.get("id"), bucket=datasource_info.get("bucket"), ), provider_type=datasource_type,