From 6529442553a0effd8661ffa0bf1ad25fe5a2b92d Mon Sep 17 00:00:00 2001 From: junjun Date: Thu, 22 May 2025 12:26:42 +0800 Subject: [PATCH] feat(datasource): Support oracle --- .../versions/e6276ddab06e_006_add_ds_field.py | 29 ++++++++ backend/apps/datasource/crud/datasource.py | 6 ++ backend/apps/datasource/models/datasource.py | 5 +- backend/apps/db/db.py | 74 ++++++++++++++++--- backend/apps/db/type.py | 13 ++++ backend/pyproject.toml | 3 +- frontend/src/views/ds/form.vue | 18 ++++- frontend/src/views/ds/index.vue | 5 +- frontend/src/views/ds/js/ds-type.ts | 15 ++-- 9 files changed, 142 insertions(+), 26 deletions(-) create mode 100644 backend/alembic/versions/e6276ddab06e_006_add_ds_field.py create mode 100644 backend/apps/db/type.py diff --git a/backend/alembic/versions/e6276ddab06e_006_add_ds_field.py b/backend/alembic/versions/e6276ddab06e_006_add_ds_field.py new file mode 100644 index 00000000..fbee3057 --- /dev/null +++ b/backend/alembic/versions/e6276ddab06e_006_add_ds_field.py @@ -0,0 +1,29 @@ +"""006_add_ds_field + +Revision ID: e6276ddab06e +Revises: 0a6f11be9be4 +Create Date: 2025-05-22 11:43:40.176878 + +""" +from alembic import op +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'e6276ddab06e' +down_revision = '0a6f11be9be4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('core_datasource', sa.Column('type_name', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('core_datasource', 'type_name') + # ### end Alembic commands ### diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 6438aa8c..87a599a6 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -15,6 +15,7 @@ from ..crud.field import delete_field_by_ds_id, update_field from ..crud.table import delete_table_by_ds_id, update_table from ..models.datasource import CoreDatasource, CreateDatasource, CoreTable, CoreField, ColumnSchema, EditObj, \ DatasourceConf +from apps.db.type import db_type_relation def get_datasource_list(session: SessionDep): @@ -41,6 +42,7 @@ def create_ds(session: SessionDep, user: CurrentUser, create_ds: CreateDatasourc # status = check_status(session, ds) ds.create_by = user.id ds.status = "Success" + ds.type_name = db_type_relation()[ds.type] record = CoreDatasource(**ds.model_dump()) session.add(record) session.flush() @@ -202,4 +204,8 @@ def preview(session: SessionDep, id: int, data: EditObj): OFFSET 0 ROWS FETCH NEXT 100 ROWS ONLY""" elif ds.type == "pg" or ds.type == "excel": sql = f"""SELECT "{'", "'.join([f.field_name for f in data.fields if f.checked])}" FROM "{conf.dbSchema}"."{data.table.table_name}" LIMIT 100""" + elif ds.type == "oracle": + sql = f"""SELECT "{'", "'.join([f.field_name for f in data.fields if f.checked])}" FROM "{conf.dbSchema}"."{data.table.table_name}" + ORDER BY "{data.fields[0].field_name}" + OFFSET 0 ROWS FETCH NEXT 100 ROWS ONLY""" return exec_sql(ds, sql) diff --git a/backend/apps/datasource/models/datasource.py b/backend/apps/datasource/models/datasource.py index 9e9d0d58..59ef6562 100644 --- a/backend/apps/datasource/models/datasource.py +++ b/backend/apps/datasource/models/datasource.py @@ -12,6 +12,7 @@ class CoreDatasource(SQLModel, table=True): name: str = Field(max_length=128, nullable=False) description: str = Field(max_length=512, nullable=True) type: str = Field(max_length=64) + type_name: str = Field(max_length=64, nullable=True) configuration: str = Field(sa_column=Column(Text)) create_time: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=True)) create_by: int = Field(sa_column=Column(BigInteger())) @@ -71,6 +72,7 @@ class DatasourceConf(BaseModel): dbSchema: str = '' filename: str = '' sheets: List = '' + mode: str = '' def to_dict(self): return { @@ -83,7 +85,8 @@ class DatasourceConf(BaseModel): "extraJdbc": self.extraJdbc, "dbSchema": self.dbSchema, "filename": self.filename, - "sheets": self.sheets + "sheets": self.sheets, + "mode": self.mode } diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index c3ad110b..5bdaaed1 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -16,19 +16,30 @@ def get_uri(ds: CoreDatasource): db_url: str if ds.type == "mysql": if conf.extraJdbc is not None and conf.extraJdbc != '': - db_url = f"mysql+pymysql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{urllib.parse.quote(conf.extraJdbc)}" + db_url = f"mysql+pymysql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{conf.extraJdbc}" else: db_url = f"mysql+pymysql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}" elif ds.type == "sqlServer": if conf.extraJdbc is not None and conf.extraJdbc != '': - db_url = f"mssql+pymssql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{urllib.parse.quote(conf.extraJdbc)}" + db_url = f"mssql+pymssql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{conf.extraJdbc}" else: db_url = f"mssql+pymssql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}" elif ds.type == "pg" or ds.type == "excel": if conf.extraJdbc is not None and conf.extraJdbc != '': - db_url = f"postgresql+psycopg2://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{urllib.parse.quote(conf.extraJdbc)}" + db_url = f"postgresql+psycopg2://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{conf.extraJdbc}" else: db_url = f"postgresql+psycopg2://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}" + elif ds.type == "oracle": + if conf.mode == "service_name": + if conf.extraJdbc is not None and conf.extraJdbc != '': + db_url = f"oracle+oracledb://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}?service_name={urllib.parse.quote(conf.database)}&{conf.extraJdbc}" + else: + db_url = f"oracle+oracledb://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}?service_name={urllib.parse.quote(conf.database)}" + else: + if conf.extraJdbc is not None and conf.extraJdbc != '': + db_url = f"oracle+oracledb://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{conf.extraJdbc}" + else: + db_url = f"oracle+oracledb://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}" else: raise 'The datasource type not support.' return db_url @@ -65,7 +76,7 @@ def get_tables(ds: CoreDatasource): FROM information_schema.TABLES WHERE - TABLE_SCHEMA = '{conf.database}'; + TABLE_SCHEMA = '{conf.database}' """ elif ds.type == "sqlServer": sql = f""" @@ -81,7 +92,7 @@ def get_tables(ds: CoreDatasource): AND ep.name = 'MS_Description' WHERE t.TABLE_TYPE IN ('BASE TABLE', 'VIEW') - AND t.TABLE_SCHEMA = '{conf.dbSchema}'; + AND t.TABLE_SCHEMA = '{conf.dbSchema}' """ elif ds.type == "pg" or ds.type == "excel": sql = """ @@ -99,7 +110,27 @@ def get_tables(ds: CoreDatasource): AND c.relkind IN ('r', 'v') AND c.relname NOT LIKE 'pg_%' AND c.relname NOT LIKE 'sql_%' - ORDER BY c.relname; + ORDER BY c.relname + """ + elif ds.type == "oracle": + sql = f""" + SELECT + t.TABLE_NAME AS "TABLE_NAME", + NVL(c.COMMENTS, '') AS "TABLE_COMMENT" + FROM ( + SELECT TABLE_NAME, 'TABLE' AS OBJECT_TYPE + FROM DBA_TABLES + WHERE OWNER = '{conf.dbSchema}' + UNION ALL + SELECT VIEW_NAME AS TABLE_NAME, 'VIEW' AS OBJECT_TYPE + FROM DBA_VIEWS + WHERE OWNER = '{conf.dbSchema}' + ) t + LEFT JOIN DBA_TAB_COMMENTS c + ON t.TABLE_NAME = c.TABLE_NAME + AND c.TABLE_TYPE = t.OBJECT_TYPE + AND c.OWNER = '{conf.dbSchema}' + ORDER BY t.TABLE_NAME """ result = session.execute(text(sql)) @@ -130,7 +161,7 @@ def get_fields(ds: CoreDatasource, table_name: str = None): WHERE TABLE_SCHEMA = '{conf.database}' """ - sql2 = f" AND TABLE_NAME = '{table_name}';" if table_name is not None and table_name != "" else ";" + sql2 = f" AND TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" sql = sql1 + sql2 elif ds.type == "sqlServer": sql1 = f""" @@ -148,7 +179,7 @@ def get_fields(ds: CoreDatasource, table_name: str = None): WHERE C.TABLE_SCHEMA = '{conf.dbSchema}' """ - sql2 = f" AND C.TABLE_NAME = '{table_name}';" if table_name is not None and table_name != "" else ";" + sql2 = f" AND C.TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" sql = sql1 + sql2 elif ds.type == "pg" or ds.type == "excel": sql1 = """ @@ -167,7 +198,32 @@ def get_fields(ds: CoreDatasource, table_name: str = None): AND a.attnum > 0 AND NOT a.attisdropped """ - sql2 = f" AND c.relname = '{table_name}';" if table_name is not None and table_name != "" else ";" + sql2 = f" AND c.relname = '{table_name}'" if table_name is not None and table_name != "" else "" + sql = sql1 + sql2 + elif ds.type == "oracle": + sql1 = f""" + SELECT + col.COLUMN_NAME AS "COLUMN_NAME", + (CASE + WHEN col.DATA_TYPE IN ('VARCHAR2', 'CHAR', 'NVARCHAR2', 'NCHAR') + THEN col.DATA_TYPE || '(' || col.DATA_LENGTH || ')' + WHEN col.DATA_TYPE = 'NUMBER' AND col.DATA_PRECISION IS NOT NULL + THEN col.DATA_TYPE || '(' || col.DATA_PRECISION || + CASE WHEN col.DATA_SCALE > 0 THEN ',' || col.DATA_SCALE END || ')' + ELSE col.DATA_TYPE + END) AS "DATA_TYPE", + NVL(com.COMMENTS, '') AS "COLUMN_COMMENT" + FROM + DBA_TAB_COLUMNS col + LEFT JOIN + DBA_COL_COMMENTS com + ON col.OWNER = com.OWNER + AND col.TABLE_NAME = com.TABLE_NAME + AND col.COLUMN_NAME = com.COLUMN_NAME + WHERE + col.OWNER = '{conf.dbSchema}' + """ + sql2 = f" AND col.TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" sql = sql1 + sql2 result = session.execute(text(sql)) diff --git a/backend/apps/db/type.py b/backend/apps/db/type.py new file mode 100644 index 00000000..55226527 --- /dev/null +++ b/backend/apps/db/type.py @@ -0,0 +1,13 @@ +# Author: Junjun +# Date: 2025/5/22 +from typing import Dict + + +def db_type_relation() -> Dict: + return { + "mysql": "MySQL", + "sqlServer": "Microsoft SQL Server", + "pg": "PostgreSQL", + "excel": "Excel/CSV", + "oracle": "Oracle" + } diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 7f11a770..b33a7151 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -32,7 +32,8 @@ dependencies = [ "pymssql (>=2.3.4,<3.0.0)", "pandas (>=2.2.3,<3.0.0)", "openpyxl (>=3.1.5,<4.0.0)", - "psycopg2 (>=2.9.10,<3.0.0)" + "psycopg2 (>=2.9.10,<3.0.0)", + "oracledb (>=3.1.1,<4.0.0)" ] [[tool.uv.index]] url = "https://pypi.tuna.tsinghua.edu.cn/simple" diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index 05f33039..c0eec9bd 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -64,6 +64,12 @@ + + + Service Name + SID + + @@ -71,6 +77,8 @@ Get Schema + Supported version: 2012+ + Supported version: 12+ @@ -143,7 +151,8 @@ const config = ref({ extraJdbc:'', dbSchema:'', filename:'', - sheets: [] + sheets: [], + mode:'service_name' }) const close = () => { @@ -176,6 +185,7 @@ const open = (item: any, editTable: boolean = false) => { config.value.dbSchema = configuration.dbSchema config.value.filename = configuration.filename config.value.sheets = configuration.sheets + config.value.mode = configuration.mode } if (editTable) { @@ -217,7 +227,8 @@ const open = (item: any, editTable: boolean = false) => { extraJdbc:'', dbSchema:'', filename:'', - sheets: [] + sheets: [], + mode:'service_name' } } dialogVisible.value = true @@ -268,7 +279,8 @@ const buildConf = () => { extraJdbc:config.value.extraJdbc, dbSchema:config.value.dbSchema, filename:config.value.filename, - sheets:config.value.sheets + sheets:config.value.sheets, + mode:config.value.mode })) } diff --git a/frontend/src/views/ds/index.vue b/frontend/src/views/ds/index.vue index b73c7f8e..06bc3ec8 100644 --- a/frontend/src/views/ds/index.vue +++ b/frontend/src/views/ds/index.vue @@ -30,11 +30,12 @@ +
{{ ds.name }}
-
{{ dsRelation[ds.type] }}
+
{{ ds.type_name }}
{{ ds.description }}
{{ datetimeFormat(ds.create_time) }}
@@ -58,6 +59,7 @@ import MysqlDs from '@/assets/svg/ds/mysql-ds.svg' import SQLServerDs from '@/assets/svg/ds/sqlServer-ds.svg' import PgDs from '@/assets/svg/ds/pg-ds.svg' import ExcelDs from '@/assets/svg/ds/Excel-ds.svg' +import OracleDs from '@/assets/svg/ds/oracle-ds.svg' import { Search, List, CreditCard } from '@element-plus/icons-vue' import { ref, onMounted } from 'vue' import DsForm from './form.vue' @@ -65,7 +67,6 @@ import { datasourceApi } from '@/api/datasource' import { datetimeFormat } from '@/utils/utils' import { ElMessageBox } from 'element-plus' import { useRouter } from 'vue-router' -import { dsRelation } from '@/views/ds/js/ds-type' const searchValue = ref('') const dsForm = ref() diff --git a/frontend/src/views/ds/js/ds-type.ts b/frontend/src/views/ds/js/ds-type.ts index abfc13ab..fa1e3af9 100644 --- a/frontend/src/views/ds/js/ds-type.ts +++ b/frontend/src/views/ds/js/ds-type.ts @@ -1,18 +1,13 @@ export const dsType = [ {label:"MySQL", value:"mysql"}, - {label:"Microsoft SQL Server", value:"sqlServer"}, {label:"PostgreSQL", value:"pg"}, + {label:"Microsoft SQL Server", value:"sqlServer"}, + {label:"Oracle", value:"oracle"}, {label:"Excel/CSV", value:"excel"}, ] export const haveSchema = [ 'sqlServer', - 'pg' -] - -export const dsRelation: any = { - "mysql": "MySQL", - "sqlServer": "Microsoft SQL Server", - "pg": "PostgreSQL", - "excel": "Excel/CSV", -} \ No newline at end of file + 'pg', + 'oracle' +] \ No newline at end of file