mirror of
https://gh-proxy.org/https://github.com/klinsmaya/SQLBot.git
synced 2026-11-08 01:06:59 +08:00
kris tan
65110d6791
Add eight regression cases for partition metadata, duplicate fields, blank names, valid hash-prefixed columns, empty results, and extra driver columns. Execute the production get_fields and ColumnSchema definitions with a mocked Hive cursor. Validation: python -m pytest backend/tests -q (11 passed).
125 lines
4.3 KiB
Python
125 lines
4.3 KiB
Python
"""Regression tests for Hive DESCRIBE field discovery (issue #1250)."""
|
|
|
|
import ast
|
|
import json
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock
|
|
|
|
import pytest
|
|
|
|
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def _load_symbol(relative_path, name, namespace):
|
|
# As in test_connection_pool_manager, avoid importing unrelated DB drivers.
|
|
path = BACKEND_DIR / relative_path
|
|
tree = ast.parse(path.read_text(encoding="utf-8"))
|
|
node = next(node for node in tree.body if getattr(node, "name", None) == name)
|
|
module = ast.Module(body=[node], type_ignores=[])
|
|
exec(compile(module, str(path), "exec"), namespace)
|
|
return namespace[name]
|
|
|
|
|
|
@pytest.fixture
|
|
def get_hive_fields():
|
|
cursor = Mock()
|
|
connection = SimpleNamespace(cursor=lambda: nullcontext(cursor))
|
|
pool = SimpleNamespace(connection=lambda: nullcontext(connection))
|
|
namespace = {
|
|
"json": json,
|
|
"CoreDatasource": SimpleNamespace,
|
|
"DatasourceConf": SimpleNamespace,
|
|
"aes_decrypt": lambda value: value,
|
|
"DB": SimpleNamespace(
|
|
get_db=lambda _: SimpleNamespace(connect_type="driver")
|
|
),
|
|
"ConnectType": SimpleNamespace(sqlalchemy="sqlalchemy"),
|
|
"get_driver_pool": lambda _: pool,
|
|
"get_field_sql": lambda *args: ("DESCRIBE sample", None, None),
|
|
}
|
|
_load_symbol("common/utils/utils.py", "equals_ignore_case", namespace)
|
|
_load_symbol("apps/datasource/models/datasource.py", "ColumnSchema", namespace)
|
|
get_fields = _load_symbol("apps/db/db.py", "get_fields", namespace)
|
|
|
|
def discover(rows):
|
|
cursor.fetchall.return_value = rows
|
|
fields = get_fields(SimpleNamespace(type="hive", configuration="{}"), "sample")
|
|
return [(field.fieldName, field.fieldType, field.fieldComment) for field in fields]
|
|
|
|
return discover
|
|
|
|
|
|
@pytest.mark.parametrize("repeat_partition", [False, True])
|
|
def test_partition_fields_are_retained_once_without_metadata(get_hive_fields, repeat_partition):
|
|
rows = [("id", "int", "identifier")]
|
|
if repeat_partition:
|
|
rows.append(("dt", "string", "partition date"))
|
|
rows.extend([
|
|
("", "", ""),
|
|
("# Partition Information", "", ""),
|
|
("# col_name", "data_type", "comment"),
|
|
("dt", "string", "partition date"),
|
|
])
|
|
|
|
assert get_hive_fields(rows) == [
|
|
("id", "int", "identifier"),
|
|
("dt", "string", "partition date"),
|
|
]
|
|
|
|
|
|
def test_duplicate_fields_preserve_first_occurrence_and_order(get_hive_fields):
|
|
assert get_hive_fields([
|
|
("id", "int", "identifier"),
|
|
("dt", "string", "first comment"),
|
|
("region", "string", "region"),
|
|
("dt", "string", "repeated comment"),
|
|
]) == [
|
|
("id", "int", "identifier"),
|
|
("dt", "string", "first comment"),
|
|
("region", "string", "region"),
|
|
]
|
|
|
|
|
|
def test_blank_names_and_padded_metadata_are_ignored(get_hive_fields):
|
|
assert get_hive_fields([
|
|
(None, None, None),
|
|
(" ", "", ""),
|
|
(" # Partition Information ", None, None),
|
|
(" # col_name ", "data_type ", "comment"),
|
|
("dt", "string", None),
|
|
]) == [("dt", "string", None)]
|
|
|
|
|
|
def test_regular_columns_preserve_names_types_and_comments(get_hive_fields):
|
|
assert get_hive_fields([
|
|
("id", "bigint", None),
|
|
("amount", "decimal(10,2)", b"amount"),
|
|
("#tag", "string", "tag"),
|
|
("# Partition Information", "string", "a real column"),
|
|
(" spaced name ", "string", "keep the original name"),
|
|
]) == [
|
|
("id", "bigint", None),
|
|
("amount", "decimal(10,2)", "amount"),
|
|
("#tag", "string", "tag"),
|
|
("# Partition Information", "string", "a real column"),
|
|
(" spaced name ", "string", "keep the original name"),
|
|
]
|
|
|
|
|
|
def test_extra_driver_columns_remain_supported(get_hive_fields):
|
|
assert get_hive_fields([
|
|
("id", "int", "identifier", "extra", 1, None),
|
|
("# Partition Information", None, None, "extra", 2, None),
|
|
("dt", "string", "partition date", "extra", 3, None),
|
|
]) == [
|
|
("id", "int", "identifier"),
|
|
("dt", "string", "partition date"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("rows", [[], [("", "", ""), ("# col_name", "data_type", "comment")]])
|
|
def test_no_real_columns_returns_empty_list(get_hive_fields, rows):
|
|
assert get_hive_fields(rows) == []
|