diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index 7efa247f0..de6ba8f80 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -100,6 +100,8 @@ jobs: trino, dremio, spark, + fabric, + sqlserver, ] uses: ./.github/workflows/test-warehouse.yml with: diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 5ed0ed99b..83fad4337 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -18,6 +18,8 @@ on: - duckdb - trino - dremio + - fabric + - sqlserver elementary-ref: type: string required: false @@ -159,6 +161,12 @@ jobs: run: | docker compose up -d --build --wait spark-thrift + - name: Start SQL Server + if: inputs.warehouse-type == 'sqlserver' + working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} + run: | + docker compose up -d --wait sqlserver + - name: Setup Python uses: actions/setup-python@v5 with: @@ -168,6 +176,14 @@ jobs: if: inputs.warehouse-type == 'spark' run: sudo apt-get install -y python3-dev libsasl2-dev gcc + - name: Install ODBC driver for SQL Server + if: inputs.warehouse-type == 'fabric' || inputs.warehouse-type == 'sqlserver' + run: | + curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc > /dev/null + curl -fsSL https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install dbt run: > pip install @@ -188,7 +204,7 @@ jobs: # This enables caching the seeded database state between runs. IS_DOCKER=false case "${{ inputs.warehouse-type }}" in - postgres|clickhouse|trino|dremio|duckdb|spark) IS_DOCKER=true ;; + postgres|clickhouse|trino|dremio|duckdb|spark|sqlserver) IS_DOCKER=true ;; esac if [ "$IS_DOCKER" = "true" ]; then diff --git a/elementary/clients/dbt/transient_errors.py b/elementary/clients/dbt/transient_errors.py index 14882ab5e..5f6cedaa2 100644 --- a/elementary/clients/dbt/transient_errors.py +++ b/elementary/clients/dbt/transient_errors.py @@ -45,6 +45,15 @@ "service unavailable", ) +_TSQL_TRANSIENT: Tuple[str, ...] = ( + "connection timed out", + "could not connect to the server", + "ssl syscall error", + "communication link failure", + "tcp provider: an existing connection was forcibly closed", + "login timeout expired", +) + _ADAPTER_PATTERNS: Dict[str, Tuple[str, ...]] = { "bigquery": ( # Streaming-buffer delay after a streaming insert. @@ -109,6 +118,8 @@ # DuckDB runs in-process; transient errors are rare. # Common patterns (connection reset, broken pipe) are in _COMMON. ), + "fabric": _TSQL_TRANSIENT, + "sqlserver": _TSQL_TRANSIENT, } # Pre-computed union of all adapter-specific patterns for the fallback path diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 0ce7dc0da..bdc2a9253 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -128,12 +128,12 @@ select distinct failed_tests.alert_id, {# Generate elementary unique id which is used to identify between tests, and set it as alert_class_id #} - coalesce(failed_tests.test_unique_id, 'None') || '.' || coalesce(failed_tests.column_name, 'None') || '.' || coalesce(failed_tests.sub_type, 'None') as alert_class_id, + {{ dbt.concat(["coalesce(failed_tests.test_unique_id, 'None')", "'.'", "coalesce(failed_tests.column_name, 'None')", "'.'", "coalesce(failed_tests.sub_type, 'None')"]) }} as alert_class_id, case when failed_tests.test_type = 'schema_change' then failed_tests.test_unique_id {# In old versions of elementary, elementary_test_results doesn't contain test_short_name, so we use dbt_test short_name. #} when tests.short_name = 'dimension_anomalies' then failed_tests.test_unique_id - else coalesce(failed_tests.test_unique_id, 'None') || '.' || coalesce(failed_tests.column_name, 'None') || '.' || coalesce(failed_tests.sub_type, 'None') + else {{ dbt.concat(["coalesce(failed_tests.test_unique_id, 'None')", "'.'", "coalesce(failed_tests.column_name, 'None')", "'.'", "coalesce(failed_tests.sub_type, 'None')"]) }} end as elementary_unique_id, failed_tests.data_issue_id, failed_tests.test_execution_id, diff --git a/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql b/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql index b10bc8ba6..1ad1ccb00 100644 --- a/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql +++ b/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql @@ -49,7 +49,7 @@ when elementary_test_results.test_type = 'schema_change' then elementary_test_results.test_unique_id {# In old versions of elementary, elementary_test_results doesn't contain test_short_name, so we use dbt_test short_name. #} when dbt_tests.short_name = 'dimension_anomalies' then elementary_test_results.test_unique_id - else coalesce(elementary_test_results.test_unique_id, 'None') || '.' || coalesce(nullif(elementary_test_results.column_name, ''), 'None') || '.' || coalesce(elementary_test_results.test_sub_type, 'None') + else {{ dbt.concat(["coalesce(elementary_test_results.test_unique_id, 'None')", "'.'", "coalesce(nullif(elementary_test_results.column_name, ''), 'None')", "'.'", "coalesce(elementary_test_results.test_sub_type, 'None')"]) }} end as elementary_unique_id, elementary_test_results.invocation_id, elementary_test_results.data_issue_id, diff --git a/elementary/monitor/dbt_project/macros/base_queries/resources.sql b/elementary/monitor/dbt_project/macros/base_queries/resources.sql index d0d819eec..b95c58bd8 100644 --- a/elementary/monitor/dbt_project/macros/base_queries/resources.sql +++ b/elementary/monitor/dbt_project/macros/base_queries/resources.sql @@ -7,10 +7,10 @@ select unique_id, name, - schema_name as schema, + schema_name as {{ elementary_cli.edr_quote_identifier('schema') }}, tags, owner as owners, - database_name as database + database_name as {{ elementary_cli.edr_quote_identifier('database') }} from dbt_models {% if exclude_elementary %} where package_name != 'elementary' @@ -31,10 +31,10 @@ unique_id, name, source_name, - schema_name AS schema, + schema_name AS {{ elementary_cli.edr_quote_identifier('schema') }}, tags, owner AS owners, - database_name as database + database_name as {{ elementary_cli.edr_quote_identifier('database') }} from dbt_sources {% if exclude_elementary %} where package_name != 'elementary' diff --git a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql index 2a021d3b5..3a9a3c186 100644 --- a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql +++ b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql @@ -21,3 +21,7 @@ {% macro duckdb__get_adapter_unique_id() %} {{ return(target.path) }} {% endmacro %} + +{% macro fabric__get_adapter_unique_id() %} + {{ return(target.server) }} +{% endmacro %} diff --git a/elementary/monitor/dbt_project/macros/get_latest_invocation.sql b/elementary/monitor/dbt_project/macros/get_latest_invocation.sql index a2590e84f..f551182f7 100644 --- a/elementary/monitor/dbt_project/macros/get_latest_invocation.sql +++ b/elementary/monitor/dbt_project/macros/get_latest_invocation.sql @@ -7,7 +7,11 @@ {% endif %} {% set get_pkg_version_query %} + {% if elementary.is_tsql() %} + select top 1 * from {{ invocations_relation }} order by generated_at desc + {% else %} select * from {{ invocations_relation }} order by generated_at desc limit 1 + {% endif %} {% endset %} {% set result = elementary.run_query(get_pkg_version_query) %} {% if not result %} diff --git a/elementary/monitor/dbt_project/macros/get_models_runs.sql b/elementary/monitor/dbt_project/macros/get_models_runs.sql index dc8851780..f17aedd27 100644 --- a/elementary/monitor/dbt_project/macros/get_models_runs.sql +++ b/elementary/monitor/dbt_project/macros/get_models_runs.sql @@ -11,7 +11,7 @@ unique_id, invocation_id, name, - schema_name as schema, + schema_name as {{ elementary_cli.edr_quote_identifier('schema') }}, status, case when status != 'success' then 0 diff --git a/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql b/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql index 6b94d1ddb..c7e664c3b 100644 --- a/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql +++ b/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql @@ -8,7 +8,7 @@ ), test_invocation as ( - select distinct invocation_id, detected_at + select {% if elementary.is_tsql() %}top 1{% endif %} distinct invocation_id, detected_at from elementary_test_results {% if invocation_id %} where invocation_id = {{ "'" ~ invocation_id ~ "'" }} @@ -16,7 +16,7 @@ where detected_at < {{ "'" ~ invocation_max_time ~ "'" }} {% endif %} order by detected_at desc - limit 1 + {% if not elementary.is_tsql() %}limit 1{% endif %} ) {% if invocations_relation %} diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index 35218e678..75b21d31b 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -2,6 +2,44 @@ {{ return(adapter.dispatch('get_test_results', 'elementary_cli')(days_back, invocations_per_test, disable_passed_test_metrics)) }} {%- endmacro -%} +{# + Shared post-processing helper: filters tests by meta, attaches sample data. + Called by both default__ and fabric__ dispatches to avoid duplicating the + Jinja processing loop. +#} +{%- macro _process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status) -%} + {% set test_results = [] %} + {% set tests = elementary.agate_to_dicts(test_results_agate) %} + + {% set filtered_tests = [] %} + {% for test in tests %} + {% set test_meta = fromjson(test.meta) %} + {% if test_meta.get("elementary", {}).get("include", true) %} + {% do filtered_tests.append(test) %} + {% endif %} + {% endfor %} + + {% for test in filtered_tests %} + {% set test_rows_sample = none %} + {% if test.invocations_rank_index == 1 %} + {% set test_type = test.test_type %} + {% set test_params = fromjson(test.test_params) %} + {% set status = test.status | lower %} + + {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} + {%- endif -%} + {% else %} + {# Null out test_results_query for non-latest invocations to save memory #} + {% do test.update({"test_results_query": none}) %} + {% endif %} + {# Adding sample data to test results #} + {% do test.update({"sample_data": test_rows_sample}) %} + {% do test_results.append(test) %} + {%- endfor -%} + {% do return(test_results) %} +{%- endmacro -%} + {%- macro default__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} {% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass'] %} {% set select_test_results %} @@ -59,8 +97,6 @@ order by test_results.elementary_unique_id, test_results.invocations_rank_index desc {%- endset -%} - {% set test_results = [] %} - {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %} {% set ordered_test_results_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_test_results) %} @@ -79,32 +115,63 @@ {% if not elementary.has_temp_table_support() %} {% do elementary.fully_drop_relation(ordered_test_results_relation) %} {% endif %} - {% set tests = elementary.agate_to_dicts(test_results_agate) %} - {% set filtered_tests = [] %} - {% for test in tests %} - {% set test_meta = fromjson(test.meta) %} - {% if test_meta.get("elementary", {}).get("include", true) %} - {% do filtered_tests.append(test) %} - {% endif %} - {% endfor %} + {% do return(elementary_cli._process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status)) %} +{%- endmacro -%} - {% for test in filtered_tests %} - {% set test_rows_sample = none %} - {% if test.invocations_rank_index == 1 %} - {% set test_type = test.test_type %} - {% set test_params = fromjson(test.test_params) %} - {% set status = test.status | lower %} +{%- macro fabric__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} + {# + T-SQL does not allow nested CTEs (WITH inside WITH). + current_tests_run_results_query already starts with WITH, so we + cannot wrap it in another CTE. Instead we materialise it into a + temp table first, then build ordered_test_results on top. + Note: sqlserver adapter inherits from fabric, so this dispatch + covers both fabric and sqlserver targets automatically. + #} + {% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass'] %} - {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} - {%- endif -%} - {% endif %} - {# Adding sample data to test results #} - {% do test.update({"sample_data": test_rows_sample}) %} - {% do test_results.append(test) %} - {%- endfor -%} - {% do return(test_results) %} + {# Step 1 – materialise the base test-results query into a temp table #} + {% set base_query %} + {{ elementary_cli.current_tests_run_results_query(days_back=days_back) }} + {% endset %} + + {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %} + {% set base_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'test_results_base', base_query) %} + + {# Step 2 – build ordered_test_results from the materialised base (no nested CTE) #} + {% set select_ordered %} + select + *, + {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp('detected_at'), elementary.edr_current_timestamp(), 'day') }} as days_diff, + row_number() over (partition by elementary_unique_id order by {{elementary.edr_cast_as_timestamp('detected_at')}} desc) as invocations_rank_index + from {{ base_relation }} + {% endset %} + + {% set ordered_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_ordered) %} + + {# Step 3 – final query: filter by invocations_per_test #} + {# ORDER BY must be here, not inside create_temp_table — T-SQL forbids ORDER BY in views/subqueries without TOP #} + {% set test_results_agate_sql %} + select * + from {{ ordered_relation }} + where invocations_rank_index <= {{ invocations_per_test }} + order by elementary_unique_id, invocations_rank_index desc + {% endset %} + + {% set valid_ids_query %} + select distinct id + from {{ ordered_relation }} + where invocations_rank_index = 1 + {% endset %} + + {% set test_results_agate = elementary.run_query(test_results_agate_sql) %} + {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %} + + {# Clean up intermediate tables #} + {% do elementary.fully_drop_relation(base_relation) %} + {% do elementary.fully_drop_relation(ordered_relation) %} + + {% do return(elementary_cli._process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status)) %} {%- endmacro -%} {%- macro clickhouse__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} @@ -162,7 +229,7 @@ CASE WHEN etr.test_type = 'schema_change' THEN etr.test_unique_id WHEN dt.short_name = 'dimension_anomalies' THEN etr.test_unique_id - ELSE coalesce(etr.test_unique_id, 'None') || '.' || coalesce(nullif(etr.column_name, ''), 'None') || '.' || coalesce(etr.test_sub_type, 'None') + ELSE {{ dbt.concat(["coalesce(etr.test_unique_id, 'None')", "'.'", "coalesce(nullif(etr.column_name, ''), 'None')", "'.'", "coalesce(etr.test_sub_type, 'None')"]) }} END AS elementary_unique_id, etr.detected_at, etr.database_name, @@ -239,31 +306,6 @@ {% if not elementary.has_temp_table_support() %} {% do elementary.fully_drop_relation(ordered_test_results_relation) %} {% endif %} - {% set tests = elementary.agate_to_dicts(test_results_agate) %} - {% set filtered_tests = [] %} - {% for test in tests %} - {% set test_meta = fromjson(test.meta) %} - {% if test_meta.get("elementary", {}).get("include", true) %} - {% do filtered_tests.append(test) %} - {% endif %} - {% endfor %} - - {% for test in filtered_tests %} - {% set test_rows_sample = none %} - {% if test.invocations_rank_index == 1 %} - {% set test_type = test.test_type %} - {% set test_params = fromjson(test.test_params) %} - {% set status = test.status | lower %} - - {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} - {%- endif -%} - {% endif %} - {# Adding sample data to test results #} - {% do test.update({"sample_data": test_rows_sample}) %} - {% do test_results.append(test) %} - {%- endfor -%} - - {% do return(test_results) %} -{%- endmacro -%} \ No newline at end of file + {% do return(elementary_cli._process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status)) %} +{%- endmacro -%} diff --git a/elementary/monitor/dbt_project/macros/test_conn.sql b/elementary/monitor/dbt_project/macros/test_conn.sql index 6f1ed9b64..ad3b52d38 100644 --- a/elementary/monitor/dbt_project/macros/test_conn.sql +++ b/elementary/monitor/dbt_project/macros/test_conn.sql @@ -6,7 +6,11 @@ {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %} {% set elementary_model_relation = api.Relation.create(elementary_database, elementary_schema, "dbt_models") %} {% set query %} + {% if elementary.is_tsql() %} + select top 10 * from {{ elementary_model_relation }} + {% else %} select * from {{ elementary_model_relation }} limit 10 + {% endif %} {% endset %} {% do elementary.run_query(query) %} {% endmacro %} diff --git a/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql b/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql new file mode 100644 index 000000000..c0cef95f0 --- /dev/null +++ b/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql @@ -0,0 +1,7 @@ +{%- macro edr_quote_identifier(identifier) -%} + {%- if elementary.is_tsql() -%} + [{{ identifier }}] + {%- else -%} + {{ identifier }} + {%- endif -%} +{%- endmacro -%} diff --git a/elementary/monitor/dbt_project/package-lock.yml b/elementary/monitor/dbt_project/package-lock.yml index 48102b18e..070158da0 100644 --- a/elementary/monitor/dbt_project/package-lock.yml +++ b/elementary/monitor/dbt_project/package-lock.yml @@ -1,6 +1,8 @@ packages: - - package: dbt-labs/dbt_utils + - name: dbt_utils + package: dbt-labs/dbt_utils version: 0.8.6 - git: https://github.com/elementary-data/dbt-data-reliability.git - revision: ab21363935c42490a60a779557ba99bed96b754c -sha1_hash: 661e08669f6a005c445ed631a333de316b58d57f + name: elementary + revision: 534afc63c75d28b87d7cbd3b222dd3ea9a980f7b +sha1_hash: cb18b7df65415901187dcf469dcd377e56c0dc70 diff --git a/elementary/monitor/dbt_project/packages.yml b/elementary/monitor/dbt_project/packages.yml index 7e8c9cea4..3a77430a4 100644 --- a/elementary/monitor/dbt_project/packages.yml +++ b/elementary/monitor/dbt_project/packages.yml @@ -2,7 +2,7 @@ packages: - package: dbt-labs/dbt_utils version: [">=0.8.0", "<0.9.0"] - git: https://github.com/elementary-data/dbt-data-reliability.git - revision: ab21363935c42490a60a779557ba99bed96b754c + revision: 534afc63c75d28b87d7cbd3b222dd3ea9a980f7b # NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this # commented, so it will be easy to access) diff --git a/pyproject.toml b/pyproject.toml index 6043c7b10..057f208c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,8 @@ dbt-trino = {version = ">=1.5.0,<2.0.0", optional = true} dbt-clickhouse = {version = ">=0.20,<2.0.0", optional = true} dbt-duckdb = {version = ">=1.5.0,<2.0.0", optional = true} dbt-dremio = {version = ">=1.5.0,<2.0.0", optional = true} +dbt-fabric = {version = ">=1.4,<2.0.0", optional = true} +dbt-sqlserver = {version = ">=1.4,<2.0.0", optional = true} [tool.poetry.extras] snowflake = ["dbt-snowflake"] bigquery = ["dbt-bigquery"] @@ -68,7 +70,9 @@ clickhouse = ["dbt-clickhouse"] trino = ["dbt-trino"] duckdb = ["dbt-duckdb"] dremio = ["dbt-dremio"] -all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-clickhouse", "dbt-athena-community", "dbt-trino", "dbt-duckdb", "dbt-dremio"] +fabric = ["dbt-fabric"] +sqlserver = ["dbt-sqlserver"] +all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-clickhouse", "dbt-athena-community", "dbt-trino", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml index 47b4c4bb7..542d31537 100644 --- a/tests/e2e_dbt_project/docker-compose.yml +++ b/tests/e2e_dbt_project/docker-compose.yml @@ -290,6 +290,22 @@ services: timeout: 5s retries: 10 + # ── SQL Server (for Fabric / SQL Server adapters) ───────────────── + sqlserver: + image: mcr.microsoft.com/mssql/server:2022-latest + container_name: sqlserver + ports: + - "127.0.0.1:1433:1433" + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: "${MSSQL_SA_PASSWORD:-Elementary123!}" + MSSQL_PID: "Developer" + healthcheck: + test: /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "${MSSQL_SA_PASSWORD:-Elementary123!}" -C -Q "SELECT 1" -b + interval: 10s + timeout: 5s + retries: 10 + networks: dremio-lakehouse: diff --git a/tests/e2e_dbt_project/models/schema.yml b/tests/e2e_dbt_project/models/schema.yml index 7934fae2a..c1eeb09b6 100644 --- a/tests/e2e_dbt_project/models/schema.yml +++ b/tests/e2e_dbt_project/models/schema.yml @@ -347,7 +347,7 @@ models: - name: groups columns: - name: group_a - data_type: "{{ 'strIng' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb') else 'CHArACTER varying' if target.type == 'redshift' else 'teXt' }}" + data_type: "{{ 'strIng' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb' or target.type == 'fabric' or target.type == 'sqlserver') else 'CHArACTER varying' if target.type == 'redshift' else 'teXt' }}" - name: group_b data_type: double - name: group_c @@ -365,7 +365,7 @@ models: - name: stats_players columns: - name: player - data_type: "{{ 'STRING' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb') else 'character varying' if target.type == 'redshift' else 'TEXT' }}" + data_type: "{{ 'STRING' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb' or target.type == 'fabric' or target.type == 'sqlserver') else 'character varying' if target.type == 'redshift' else 'TEXT' }}" - name: goals data_type: BOOLEAN - name: coffee_cups_consumed diff --git a/tests/profiles/profiles.yml.j2 b/tests/profiles/profiles.yml.j2 index 77138bf93..3543496c6 100644 --- a/tests/profiles/profiles.yml.j2 +++ b/tests/profiles/profiles.yml.j2 @@ -60,8 +60,36 @@ elementary_tests: file_format: delta threads: 4 + sqlserver: &sqlserver + type: sqlserver + driver: "ODBC Driver 18 for SQL Server" + server: 127.0.0.1 + port: 1433 + database: master + schema: {{ schema_name }} + user: sa + password: "Elementary123!" + encrypt: false + trust_cert: true + threads: 4 + # ── Cloud targets (secrets substituted at CI time) ───────────────── + fabric: &fabric + type: fabric + driver: "ODBC Driver 18 for SQL Server" + server: {{ fabric_server | toyaml }} + port: 1433 + database: {{ fabric_database | toyaml }} + schema: {{ schema_name }} + authentication: ServicePrincipal + tenant_id: {{ fabric_tenant_id | toyaml }} + client_id: {{ fabric_client_id | toyaml }} + client_secret: {{ fabric_client_secret | toyaml }} + encrypt: true + trust_cert: false + threads: 4 + snowflake: &snowflake type: snowflake account: {{ snowflake_account | toyaml }} @@ -120,7 +148,7 @@ elementary_tests: elementary: target: postgres outputs: -{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %} +{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'fabric', 'sqlserver', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %} {%- for t in targets %} {{ t }}: <<: *{{ t }}