feat:add seekdb (#66)

* config adapt revert

* ci test

* fix mysql migration test

* fix

* fix

* lint fix

* fix ob config

* fix

* fix

* fix

* test over

* test

* fix

* fix

* fix style

* test over

* retain gin for pg

* gin for pg

* uuid defalut in versions

* ci test

* ci test

* fix

* fix

* fix

* fix

* pg josnb

* fix

* fix

* add seekdb

* test over

* test over

.github/workflows/api-tests.yml

@@ -1,9 +1,7 @@
 name: Run Pytest
 
 on:
-  push:
+  workflow_call:
-    branches:
-      - mysql-adapt
 
 concurrency:
   group: api-tests-${{ github.head_ref || github.run_id }}

.github/workflows/db-migration-test.yml

@@ -1,9 +1,7 @@
 name: DB Migration Test
 
 on:
-  push:
+  workflow_call:
-    branches:
-      - mysql-adapt
 
 concurrency:
   group: db-migration-test-${{ github.ref }}

.github/workflows/style.yml

@@ -1,9 +1,7 @@
 name: Style check
 
 on:
-  push:
+  workflow_call:
-    branches:
-      - mysql-adapt
 
 concurrency:
   group: style-${{ github.head_ref || github.run_id }}

.gitignore

@@ -187,6 +187,7 @@ docker/volumes/oceanbase/*
 docker/volumes/plugin_daemon/*
 docker/volumes/matrixone/*
 docker/volumes/mysql/*
+docker/volumes/seekdb/*
 !docker/volumes/oceanbase/init.d
 
 docker/nginx/conf.d/default.conf

api/.env.example

@@ -187,6 +187,7 @@ OCEANBASE_VECTOR_DATABASE=test
 OCEANBASE_MEMORY_LIMIT=6G
 OCEANBASE_ENABLE_HYBRID_SEARCH=false
 OCEANBASE_FULLTEXT_PARSER=ik
+SEEKDB_MEMORY_LIMIT=2G
 
 # Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode
 QDRANT_URL=http://localhost:6333

docker/.env.example

@@ -224,16 +224,20 @@ NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false
 
 # ------------------------------
 # Database Configuration
-# The database uses PostgreSQL or MySQL. Please use the public schema.
+# The database uses PostgreSQL or MySQL. OceanBase and SeekDB are also supported. Please use the public schema.
-# It is consistent with the configuration in the databse service below.
+# It is consistent with the configuration in the database service below.
+# You can adjust the database configuration according to your needs.
 # ------------------------------
 
+# Database type, supported values are `postgresql` and `mysql`
 DB_TYPE=postgresql
+
 DB_USERNAME=postgres
 DB_PASSWORD=difyai123456
 DB_HOST=db_postgres
 DB_PORT=5432
 DB_DATABASE=dify
+
 # The size of the database connection pool.
 # The default is 30 connections, which can be appropriately increased.
 SQLALCHEMY_POOL_SIZE=30
@@ -523,9 +527,10 @@ WEAVIATE_ENDPOINT=http://weaviate:8080
 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
 WEAVIATE_GRPC_ENDPOINT=grpc://weaviate:50051
 
-# For OceanBase metadata database configuration, available when `DB_TYPE` is `mysql` and `COMPOSE_PROFILES` includes `oceanbase`
+# For OceanBase metadata database configuration, available when `DB_TYPE` is `mysql` and `COMPOSE_PROFILES` includes `oceanbase`.
 # For OceanBase vector database configuration, available when `VECTOR_STORE` is `oceanbase`
-# If you want to use OceanBase as both vector database and metadata database, you need to set `DB_TYPE` to `mysql` and `COMPOSE_PROFILES` is `oceanbase`.
+# If you want to use OceanBase as both vector database and metadata database, you need to set `DB_TYPE` to `mysql`, `COMPOSE_PROFILES` is `oceanbase`, and set Database Configuration is the same as the vector database.
+# SeekDB is the lite version of OceanBase and shares the connection configuration with OceanBase.
 OCEANBASE_VECTOR_HOST=oceanbase
 OCEANBASE_VECTOR_PORT=2881
 OCEANBASE_VECTOR_USER=root@test
@@ -537,6 +542,7 @@ OCEANBASE_ENABLE_HYBRID_SEARCH=false
 # For OceanBase vector database, built-in fulltext parsers are `ngram`, `beng`, `space`, `ngram2`, `ik`
 # For OceanBase vector database, external fulltext parsers (require plugin installation) are `japanese_ftparser`, `thai_ftparser`
 OCEANBASE_FULLTEXT_PARSER=ik
+SEEKDB_MEMORY_LIMIT=2G
 
 # The Qdrant endpoint URL. Only available when VECTOR_STORE is `qdrant`.
 QDRANT_URL=http://qdrant:6333

docker/docker-compose-template.yaml

@@ -26,6 +26,9 @@ services:
       oceanbase:
         condition: service_healthy
         required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
     volumes:
@@ -60,6 +63,9 @@ services:
       oceanbase:
         condition: service_healthy
         required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
     volumes:
@@ -89,6 +95,9 @@ services:
       oceanbase:
         condition: service_healthy
         required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
     networks:
@@ -301,6 +310,9 @@ services:
       oceanbase:
         condition: service_healthy
         required: false
+      seekdb:
+        condition: service_healthy
+        required: false
 
   # ssrf_proxy server
   # for more information, please refer to
@@ -441,13 +453,38 @@ services:
       test:
         [
           "CMD-SHELL",
-          'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD} -e "SELECT 1;"',
+          'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"',
         ]
       interval: 10s
       retries: 30
       start_period: 30s
       timeout: 10s
 
+  # SeekDB vector database
+  seekdb: 
+    image: oceanbase/seekdb:latest
+    container_name: seekdb
+    profiles:
+      - seekdb
+    restart: always
+    volumes:
+      - ./volumes/seekdb:/var/lib/oceanbase
+    environment:
+      ROOT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456}
+      MEMORY_LIMIT: ${SEEKDB_MEMORY_LIMIT:-2G}
+      REPORTER: dify-ai-seekdb
+    ports:
+      - "${OCEANBASE_VECTOR_PORT:-2881}:2881"
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          'mysql -h127.0.0.1 -P2881 -uroot -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"',
+        ]
+      interval: 5s
+      retries: 60
+      timeout: 5s
+
   # Qdrant vector store.
   # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.)
   qdrant:

docker/docker-compose.yaml

@@ -173,6 +173,7 @@ x-shared-env: &shared-api-worker-env
   OCEANBASE_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G}
   OCEANBASE_ENABLE_HYBRID_SEARCH: ${OCEANBASE_ENABLE_HYBRID_SEARCH:-false}
   OCEANBASE_FULLTEXT_PARSER: ${OCEANBASE_FULLTEXT_PARSER:-ik}
+  SEEKDB_MEMORY_LIMIT: ${SEEKDB_MEMORY_LIMIT:-2G}
   QDRANT_URL: ${QDRANT_URL:-http://qdrant:6333}
   QDRANT_API_KEY: ${QDRANT_API_KEY:-difyai123456}
   QDRANT_CLIENT_TIMEOUT: ${QDRANT_CLIENT_TIMEOUT:-20}
@@ -658,6 +659,9 @@ services:
       oceanbase:
         condition: service_healthy
         required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
     volumes:
@@ -692,6 +696,9 @@ services:
       oceanbase:
         condition: service_healthy
         required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
     volumes:
@@ -721,6 +728,9 @@ services:
       oceanbase:
         condition: service_healthy
         required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
     networks:
@@ -933,6 +943,9 @@ services:
       oceanbase:
         condition: service_healthy
         required: false
+      seekdb:
+        condition: service_healthy
+        required: false
 
   # ssrf_proxy server
   # for more information, please refer to
@@ -1073,13 +1086,38 @@ services:
       test:
         [
           "CMD-SHELL",
-          'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD} -e "SELECT 1;"',
+          'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"',
         ]
       interval: 10s
       retries: 30
       start_period: 30s
       timeout: 10s
 
+  # SeekDB vector database
+  seekdb: 
+    image: oceanbase/seekdb:latest
+    container_name: seekdb
+    profiles:
+      - seekdb
+    restart: always
+    volumes:
+      - ./volumes/seekdb:/var/lib/oceanbase
+    environment:
+      ROOT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456}
+      MEMORY_LIMIT: ${SEEKDB_MEMORY_LIMIT:-2G}
+      REPORTER: dify-ai-seekdb
+    ports:
+      - "${OCEANBASE_VECTOR_PORT:-2881}:2881"
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          'mysql -h127.0.0.1 -P2881 -uroot -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"',
+        ]
+      interval: 5s
+      retries: 60
+      timeout: 5s
+
   # Qdrant vector store.
   # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.)
   qdrant: