From 5322f3bbd40edfbdfab28ca383c06570f340345f Mon Sep 17 00:00:00 2001 From: longbingljw Date: Sun, 16 Nov 2025 18:39:35 +0800 Subject: [PATCH] feat:add seekdb (#66) * config adapt revert * ci test * fix mysql migration test * fix * fix * lint fix * fix ob config * fix * fix * fix * test over * test * fix * fix * fix style * test over * retain gin for pg * gin for pg * uuid defalut in versions * ci test * ci test * fix * fix * fix * fix * pg josnb * fix * fix * add seekdb * test over * test over --- .github/workflows/api-tests.yml | 4 +-- .github/workflows/db-migration-test.yml | 4 +-- .github/workflows/style.yml | 4 +-- .gitignore | 1 + api/.env.example | 1 + docker/.env.example | 14 ++++++--- docker/docker-compose-template.yaml | 39 +++++++++++++++++++++++- docker/docker-compose.yaml | 40 ++++++++++++++++++++++++- 8 files changed, 92 insertions(+), 15 deletions(-) diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 7231e26ff8..557d747a8c 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -1,9 +1,7 @@ name: Run Pytest on: - push: - branches: - - mysql-adapt + workflow_call: concurrency: group: api-tests-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/db-migration-test.yml b/.github/workflows/db-migration-test.yml index 526e1ddbac..101d973466 100644 --- a/.github/workflows/db-migration-test.yml +++ b/.github/workflows/db-migration-test.yml @@ -1,9 +1,7 @@ name: DB Migration Test on: - push: - branches: - - mysql-adapt + workflow_call: concurrency: group: db-migration-test-${{ github.ref }} diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 2fe629b87b..e652657705 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -1,9 +1,7 @@ name: Style check on: - push: - branches: - - mysql-adapt + workflow_call: concurrency: group: style-${{ github.head_ref || github.run_id }} diff --git a/.gitignore b/.gitignore index 38343ec04a..79ba44b207 100644 --- a/.gitignore +++ b/.gitignore @@ -187,6 +187,7 @@ docker/volumes/oceanbase/* docker/volumes/plugin_daemon/* docker/volumes/matrixone/* docker/volumes/mysql/* +docker/volumes/seekdb/* !docker/volumes/oceanbase/init.d docker/nginx/conf.d/default.conf diff --git a/api/.env.example b/api/.env.example index 607f17d7bc..22415ee3ce 100644 --- a/api/.env.example +++ b/api/.env.example @@ -187,6 +187,7 @@ OCEANBASE_VECTOR_DATABASE=test OCEANBASE_MEMORY_LIMIT=6G OCEANBASE_ENABLE_HYBRID_SEARCH=false OCEANBASE_FULLTEXT_PARSER=ik +SEEKDB_MEMORY_LIMIT=2G # Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode QDRANT_URL=http://localhost:6333 diff --git a/docker/.env.example b/docker/.env.example index 6075940feb..17925a791e 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -224,16 +224,20 @@ NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false # ------------------------------ # Database Configuration -# The database uses PostgreSQL or MySQL. Please use the public schema. -# It is consistent with the configuration in the databse service below. +# The database uses PostgreSQL or MySQL. OceanBase and SeekDB are also supported. Please use the public schema. +# It is consistent with the configuration in the database service below. +# You can adjust the database configuration according to your needs. # ------------------------------ +# Database type, supported values are `postgresql` and `mysql` DB_TYPE=postgresql + DB_USERNAME=postgres DB_PASSWORD=difyai123456 DB_HOST=db_postgres DB_PORT=5432 DB_DATABASE=dify + # The size of the database connection pool. # The default is 30 connections, which can be appropriately increased. SQLALCHEMY_POOL_SIZE=30 @@ -523,9 +527,10 @@ WEAVIATE_ENDPOINT=http://weaviate:8080 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih WEAVIATE_GRPC_ENDPOINT=grpc://weaviate:50051 -# For OceanBase metadata database configuration, available when `DB_TYPE` is `mysql` and `COMPOSE_PROFILES` includes `oceanbase` +# For OceanBase metadata database configuration, available when `DB_TYPE` is `mysql` and `COMPOSE_PROFILES` includes `oceanbase`. # For OceanBase vector database configuration, available when `VECTOR_STORE` is `oceanbase` -# If you want to use OceanBase as both vector database and metadata database, you need to set `DB_TYPE` to `mysql` and `COMPOSE_PROFILES` is `oceanbase`. +# If you want to use OceanBase as both vector database and metadata database, you need to set `DB_TYPE` to `mysql`, `COMPOSE_PROFILES` is `oceanbase`, and set Database Configuration is the same as the vector database. +# SeekDB is the lite version of OceanBase and shares the connection configuration with OceanBase. OCEANBASE_VECTOR_HOST=oceanbase OCEANBASE_VECTOR_PORT=2881 OCEANBASE_VECTOR_USER=root@test @@ -537,6 +542,7 @@ OCEANBASE_ENABLE_HYBRID_SEARCH=false # For OceanBase vector database, built-in fulltext parsers are `ngram`, `beng`, `space`, `ngram2`, `ik` # For OceanBase vector database, external fulltext parsers (require plugin installation) are `japanese_ftparser`, `thai_ftparser` OCEANBASE_FULLTEXT_PARSER=ik +SEEKDB_MEMORY_LIMIT=2G # The Qdrant endpoint URL. Only available when VECTOR_STORE is `qdrant`. QDRANT_URL=http://qdrant:6333 diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 67158c3ef7..beb9a2ea75 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -26,6 +26,9 @@ services: oceanbase: condition: service_healthy required: false + seekdb: + condition: service_healthy + required: false redis: condition: service_started volumes: @@ -60,6 +63,9 @@ services: oceanbase: condition: service_healthy required: false + seekdb: + condition: service_healthy + required: false redis: condition: service_started volumes: @@ -89,6 +95,9 @@ services: oceanbase: condition: service_healthy required: false + seekdb: + condition: service_healthy + required: false redis: condition: service_started networks: @@ -301,6 +310,9 @@ services: oceanbase: condition: service_healthy required: false + seekdb: + condition: service_healthy + required: false # ssrf_proxy server # for more information, please refer to @@ -441,13 +453,38 @@ services: test: [ "CMD-SHELL", - 'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD} -e "SELECT 1;"', + 'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"', ] interval: 10s retries: 30 start_period: 30s timeout: 10s + # SeekDB vector database + seekdb: + image: oceanbase/seekdb:latest + container_name: seekdb + profiles: + - seekdb + restart: always + volumes: + - ./volumes/seekdb:/var/lib/oceanbase + environment: + ROOT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} + MEMORY_LIMIT: ${SEEKDB_MEMORY_LIMIT:-2G} + REPORTER: dify-ai-seekdb + ports: + - "${OCEANBASE_VECTOR_PORT:-2881}:2881" + healthcheck: + test: + [ + "CMD-SHELL", + 'mysql -h127.0.0.1 -P2881 -uroot -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"', + ] + interval: 5s + retries: 60 + timeout: 5s + # Qdrant vector store. # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.) qdrant: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 8b73152a89..081e438cd3 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -173,6 +173,7 @@ x-shared-env: &shared-api-worker-env OCEANBASE_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G} OCEANBASE_ENABLE_HYBRID_SEARCH: ${OCEANBASE_ENABLE_HYBRID_SEARCH:-false} OCEANBASE_FULLTEXT_PARSER: ${OCEANBASE_FULLTEXT_PARSER:-ik} + SEEKDB_MEMORY_LIMIT: ${SEEKDB_MEMORY_LIMIT:-2G} QDRANT_URL: ${QDRANT_URL:-http://qdrant:6333} QDRANT_API_KEY: ${QDRANT_API_KEY:-difyai123456} QDRANT_CLIENT_TIMEOUT: ${QDRANT_CLIENT_TIMEOUT:-20} @@ -658,6 +659,9 @@ services: oceanbase: condition: service_healthy required: false + seekdb: + condition: service_healthy + required: false redis: condition: service_started volumes: @@ -692,6 +696,9 @@ services: oceanbase: condition: service_healthy required: false + seekdb: + condition: service_healthy + required: false redis: condition: service_started volumes: @@ -721,6 +728,9 @@ services: oceanbase: condition: service_healthy required: false + seekdb: + condition: service_healthy + required: false redis: condition: service_started networks: @@ -933,6 +943,9 @@ services: oceanbase: condition: service_healthy required: false + seekdb: + condition: service_healthy + required: false # ssrf_proxy server # for more information, please refer to @@ -1073,13 +1086,38 @@ services: test: [ "CMD-SHELL", - 'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD} -e "SELECT 1;"', + 'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"', ] interval: 10s retries: 30 start_period: 30s timeout: 10s + # SeekDB vector database + seekdb: + image: oceanbase/seekdb:latest + container_name: seekdb + profiles: + - seekdb + restart: always + volumes: + - ./volumes/seekdb:/var/lib/oceanbase + environment: + ROOT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} + MEMORY_LIMIT: ${SEEKDB_MEMORY_LIMIT:-2G} + REPORTER: dify-ai-seekdb + ports: + - "${OCEANBASE_VECTOR_PORT:-2881}:2881" + healthcheck: + test: + [ + "CMD-SHELL", + 'mysql -h127.0.0.1 -P2881 -uroot -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"', + ] + interval: 5s + retries: 60 + timeout: 5s + # Qdrant vector store. # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.) qdrant: