From f586aec67f99431db4068522b38e337f29dba1bd Mon Sep 17 00:00:00 2001 From: junjun Date: Thu, 14 Aug 2025 17:58:01 +0800 Subject: [PATCH] feat: support ClickHouse datasource --- backend/apps/datasource/crud/datasource.py | 4 ++++ backend/apps/db/constant.py | 1 + backend/apps/db/db.py | 26 ++++++++++++++++++++- backend/apps/db/type.py | 3 ++- backend/pyproject.toml | 1 + backend/template.yaml | 2 +- frontend/src/assets/datasource/icon_ck.png | Bin 0 -> 286 bytes frontend/src/views/ds/js/ds-type.ts | 3 +++ 8 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 frontend/src/assets/datasource/icon_ck.png diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 6293bca3..db547011 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -293,6 +293,10 @@ def preview(session: SessionDep, current_user: CurrentUser, id: int, data: Table {where} ORDER BY "{fields[0]}" OFFSET 0 ROWS FETCH NEXT 100 ROWS ONLY""" + elif ds.type == "ck": + sql = f"""SELECT "{'", "'.join(fields)}" FROM "{data.table.table_name}" + {where} + LIMIT 100""" return exec_sql(ds, sql) diff --git a/backend/apps/db/constant.py b/backend/apps/db/constant.py index 51126272..18141c6e 100644 --- a/backend/apps/db/constant.py +++ b/backend/apps/db/constant.py @@ -10,6 +10,7 @@ class DB(Enum): pg = ('pg', '"', '"') excel = ('excel', '"', '"') oracle = ('oracle', '"', '"') + ck = ('ClickHouse', '"', '"') def __init__(self, type, prefix, suffix): self.type = type diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 680b895f..8584f1fc 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -46,6 +46,11 @@ def get_uri_from_config(type: str, conf: DatasourceConf) -> str: db_url = f"oracle+oracledb://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{conf.database}?{conf.extraJdbc}" else: db_url = f"oracle+oracledb://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{conf.database}" + elif type == "ck": + if conf.extraJdbc is not None and conf.extraJdbc != '': + db_url = f"clickhouse+http://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{conf.database}?{conf.extraJdbc}" + else: + db_url = f"clickhouse+http://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{conf.database}" else: raise 'The datasource type not support.' return db_url @@ -67,7 +72,7 @@ def get_engine(ds: CoreDatasource, timeout: int = 0) -> Engine: elif ds.type == 'oracle': engine = create_engine(get_uri(ds), pool_timeout=conf.timeout) - else: + else: # mysql, ck engine = create_engine(get_uri(ds), connect_args={"connect_timeout": conf.timeout}, pool_timeout=conf.timeout) return engine @@ -144,6 +149,14 @@ def get_tables(ds: CoreDatasource): AND c.OWNER = '{conf.dbSchema}' ORDER BY t.TABLE_NAME """ + elif ds.type == "ck": + sql = f""" + SELECT name, engine + FROM system.tables + WHERE database = '{conf.database}' + AND engine NOT IN ('Dictionary') + ORDER BY name + """ with session.execute(text(sql)) as result: res = result.fetchall() res_list = [TableSchema(*item) for item in res] @@ -226,6 +239,17 @@ def get_fields(ds: CoreDatasource, table_name: str = None): """ sql2 = f" AND col.TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" sql = sql1 + sql2 + elif ds.type == "ck": + sql1 = f""" + SELECT + name AS COLUMN_NAME, + type AS DATA_TYPE, + comment AS COLUMN_COMMENT + FROM system.columns + WHERE database = '{conf.database}' + """ + sql2 = f" AND table = '{table_name}'" if table_name is not None and table_name != "" else "" + sql = sql1 + sql2 with session.execute(text(sql)) as result: res = result.fetchall() diff --git a/backend/apps/db/type.py b/backend/apps/db/type.py index 55226527..e7726ba2 100644 --- a/backend/apps/db/type.py +++ b/backend/apps/db/type.py @@ -9,5 +9,6 @@ def db_type_relation() -> Dict: "sqlServer": "Microsoft SQL Server", "pg": "PostgreSQL", "excel": "Excel/CSV", - "oracle": "Oracle" + "oracle": "Oracle", + "ck": "ClickHouse" } diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 9773a519..e4b2f1b0 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ "xlsxwriter>=3.2.5", "python-calamine>=0.4.0", "xlrd>=2.0.2", + "clickhouse-sqlalchemy>=0.3.2", ] [[tool.uv.index]] name = "default" diff --git a/backend/template.yaml b/backend/template.yaml index f5a9dbf9..2b766c3a 100644 --- a/backend/template.yaml +++ b/backend/template.yaml @@ -27,7 +27,7 @@ template: - SQL查询的字段若是函数字段，如 COUNT(),CAST() 等，必须加上别名 - 计算占比，百分比类型字段，保留两位小数，以%结尾。 - 生成SQL时，必须避免关键字冲突。 - - 如数据库引擎是 PostgreSQL、Oracle，则在schema、表名、字段名、别名外层加双引号； + - 如数据库引擎是 PostgreSQL、Oracle、ClickHouse，则在schema、表名、字段名、别名外层加双引号； - 如数据库引擎是 MySQL，则在表名、字段名、别名外层加反引号； - 如数据库引擎是 Microsoft SQL Server，则在schema、表名、字段名、别名外层加方括号。 - 以PostgreSQL为例，查询Schema为TEST表TABLE下所有数据，则生成的SQL为： diff --git a/frontend/src/assets/datasource/icon_ck.png b/frontend/src/assets/datasource/icon_ck.png new file mode 100644 index 0000000000000000000000000000000000000000..b0d09c8339d21907e6c537931a76f00f9d4d1fcb GIT binary patch literal 286 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0I1SD0tpLGH$&H|6fVg?4jBOuH;Rhv&5C@5Is z8c`CQpH@FM?f2(8-u9Qwum2CConGa-iVde*gTe~ HDWM4f9erw( literal 0 HcmV?d00001 diff --git a/frontend/src/views/ds/js/ds-type.ts b/frontend/src/views/ds/js/ds-type.ts index ab10879b..267728a5 100644 --- a/frontend/src/views/ds/js/ds-type.ts +++ b/frontend/src/views/ds/js/ds-type.ts @@ -3,6 +3,7 @@ import excel from '@/assets/datasource/icon_excel.png' import oracle from '@/assets/datasource/icon_oracle.png' import pg from '@/assets/datasource/icon_PostgreSQL.png' import sqlServer from '@/assets/datasource/icon_SQL_Server.png' +import ck from '@/assets/datasource/icon_ck.png' import { i18n } from '@/i18n' const t = i18n.global.t @@ -12,6 +13,7 @@ export const dsType = [ { label: 'Oracle', value: 'oracle' }, { label: 'PostgreSQL', value: 'pg' }, { label: 'SQL Server', value: 'sqlServer' }, + { label: 'ClickHouse', value: 'ck' }, ] export const dsTypeWithImg = [ @@ -20,6 +22,7 @@ export const dsTypeWithImg = [ { name: 'Oracle', type: 'oracle', img: oracle }, { name: 'PostgreSQL', type: 'pg', img: pg }, { name: 'SQL Server', type: 'sqlServer', img: sqlServer }, + { name: 'ClickHouse', type: 'ck', img: ck }, ] export const haveSchema = ['sqlServer', 'pg', 'oracle']