Merge branch 'dbt-msft:master' into seed-logic-fix

dbt-msft · Sep 3, 2024 · 88c4ff1 · 88c4ff1
2 parents 8014117 + 024e202
commit 88c4ff1
Show file tree

Hide file tree

Showing 12 changed files with 450 additions and 18 deletions.
diff --git a/.github/workflows/integration-tests-sqlserver.yml b/.github/workflows/integration-tests-sqlserver.yml
@@ -35,7 +35,7 @@ jobs:
           DBT_TEST_USER_3: DBT_TEST_USER_3
           COLLATION: ${{ matrix.collation }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         run: pip install -r dev_requirements.txt

diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml
@@ -20,7 +20,7 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Log in to the Container registry
         uses: docker/login-action@v2.1.0
@@ -50,7 +50,7 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Log in to the Container registry
         uses: docker/login-action@v2.1.0

diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml
@@ -11,7 +11,7 @@ jobs:
     name: Release new version
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-python@v4
         with:

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -28,7 +28,7 @@ jobs:
         password: ${{ secrets.github_token }}
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         run: pip install -r dev_requirements.txt

diff --git a/dbt/adapters/sqlserver/__version__.py b/dbt/adapters/sqlserver/__version__.py
@@ -1 +1 @@
-version = "1.8.0"
+version = "1.8.0rc1"
diff --git a/dbt/include/sqlserver/macros/adapter/indexes.sql b/dbt/include/sqlserver/macros/adapter/indexes.sql
@@ -108,6 +108,7 @@
         inner join sys.tables {{ information_schema_hints() }}
         on sys.indexes.object_id = sys.tables.object_id
         where sys.indexes.[name] is not null
+        and SCHEMA_NAME(sys.tables.schema_id) = '{{ this.schema }}'
         and sys.tables.[name] = '{{ this.table }}'
         for xml path('')
     ); exec sp_executesql @drop_remaining_indexes_last;

diff --git a/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql b/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql
@@ -59,6 +59,7 @@
     {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}
     {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
 
+    {% do to_drop.append(temp_relation) %}
   {% endif %}
 
   {% call statement("main") %}

diff --git a/dbt/include/sqlserver/macros/materializations/tests.sql b/dbt/include/sqlserver/macros/materializations/tests.sql
@@ -0,0 +1,46 @@
+{% macro sqlserver__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
+
+  -- Create target schema if it does not
+  USE [{{ target.database }}];
+  IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ target.schema }}')
+  BEGIN
+    EXEC('CREATE SCHEMA [{{ target.schema }}]')
+  END
+
+  {% set with_statement_pattern = 'with .+ as\s*\(' %}
+  {% set re = modules.re %}
+  {% set is_match = re.search(with_statement_pattern, main_sql, re.IGNORECASE) %}
+
+  {% if is_match %}
+    {% set testview %}
+      [{{ target.schema }}.testview_{{ range(1300, 19000) | random }}]
+    {% endset %}
+
+    {% set sql = main_sql.replace("'", "''")%}
+    EXEC('create view {{testview}} as {{ sql }};')
+    select
+      {{ "top (" ~ limit ~ ')' if limit != none }}
+      {{ fail_calc }} as failures,
+      case when {{ fail_calc }} {{ warn_if }}
+        then 'true' else 'false' end as should_warn,
+      case when {{ fail_calc }} {{ error_if }}
+        then 'true' else 'false' end as should_error
+    from (
+      select * from {{testview}}
+    ) dbt_internal_test;
+
+    EXEC('drop view {{testview}};')
+
+  {% else -%}
+    select
+      {{ "top (" ~ limit ~ ')' if limit != none }}
+      {{ fail_calc }} as failures,
+      case when {{ fail_calc }} {{ warn_if }}
+        then 'true' else 'false' end as should_warn,
+      case when {{ fail_calc }} {{ error_if }}
+        then 'true' else 'false' end as should_error
+    from (
+      {{ main_sql }}
+    ) dbt_internal_test
+  {%- endif -%}
+{%- endmacro %}
diff --git a/tests/functional/adapter/mssql/test_index.py b/tests/functional/adapter/mssql/test_index.py
@@ -54,6 +54,18 @@
   select * from {{ ref('raw_data') }}
 """
 
+drop_schema_model = """
+{{
+  config({
+  "materialized": 'table',
+        "post-hook": [
+            "{{ drop_all_indexes_on_table() }}",
+        ]
+  })
+}}
+select * from {{ ref('raw_data') }}
+"""
+
 base_validation = """
 with base_query AS (
 select i.[name] as index_name,
@@ -107,6 +119,21 @@
 """
 )
 
+other_index_count = (
+    base_validation
+    + """
+SELECT
+  *
+FROM
+  base_query
+WHERE
+  schema_name='{schema_name}'
+  AND
+  table_view='{schema_name}.{table_name}'
+
+"""
+)
+
 
 class TestIndex:
     @pytest.fixture(scope="class")
@@ -143,3 +170,77 @@ def test_create_index(self, project):
             "Nonclustered unique index": 4,
         }
         assert schema_dict == expected
+
+
+class TestIndexDropsOnlySchema:
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {"name": "generic_tests"}
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "raw_data.csv": index_seed_csv,
+            "schema.yml": index_schema_base_yml,
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "index_model.sql": drop_schema_model,
+            "index_ccs_model.sql": model_sql_ccs,
+            "schema.yml": model_yml,
+        }
+
+    def create_table_and_index_other_schema(self, project):
+        _schema = project.test_schema + "other"
+        create_sql = f"""
+        USE [{project.database}];
+        IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{_schema}')
+        BEGIN
+        EXEC('CREATE SCHEMA [{_schema}]')
+        END
+        """
+
+        create_table = f"""
+        CREATE TABLE {_schema}.index_model (
+        IDCOL BIGINT
+        )
+        """
+
+        create_index = f"""
+        CREATE INDEX sample_schema ON {_schema}.index_model (IDCOL)
+        """
+        with get_connection(project.adapter):
+            project.adapter.execute(create_sql, fetch=True)
+            project.adapter.execute(create_table)
+            project.adapter.execute(create_index)
+
+    def drop_schema_artifacts(self, project):
+        _schema = project.test_schema + "other"
+        drop_index = f"DROP INDEX IF EXISTS sample_schema ON {_schema}.index_model"
+        drop_table = f"DROP TABLE IF EXISTS {_schema}.index_model"
+        drop_schema = f"DROP SCHEMA IF EXISTS {_schema}"
+
+        with get_connection(project.adapter):
+            project.adapter.execute(drop_index, fetch=True)
+            project.adapter.execute(drop_table)
+            project.adapter.execute(drop_schema)
+
+    def validate_other_schema(self, project):
+        with get_connection(project.adapter):
+            result, table = project.adapter.execute(
+                other_index_count.format(
+                    schema_name=project.test_schema + "other", table_name="index_model"
+                ),
+                fetch=True,
+            )
+
+        assert len(table.rows) == 1
+
+    def test_create_index(self, project):
+        self.create_table_and_index_other_schema(project)
+        run_dbt(["seed"])
+        run_dbt(["run"])
+        self.validate_other_schema(project)
+        self.drop_schema_artifacts(project)
diff --git a/tests/functional/adapter/mssql/test_materialize_change.py b/tests/functional/adapter/mssql/test_materialize_change.py
@@ -0,0 +1,62 @@
+import pytest
+from dbt.tests.util import get_connection, run_dbt
+
+model_sql = """
+SELECT 1 AS data
+"""
+
+table_mat = """
+{{
+  config({
+  "materialized": 'table'
+  })
+}}
+SELECT 1 AS data
+"""
+
+view_mat = """
+{{
+  config({
+  "materialized": 'view'
+  })
+}}
+SELECT 1 AS data
+"""
+
+schema = """
+version: 2
+models:
+  - name: mat_object
+"""
+
+
+class BaseTableView:
+    def create_object(self, project, sql):
+        with get_connection(project.adapter):
+            project.adapter.execute(sql, fetch=True)
+
+
+class TestTabletoView(BaseTableView):
+    """Test if changing from a table object to a view object correctly replaces"""
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"mat_object.sql": view_mat, "schema.yml": schema}
+
+    def test_passes(self, project):
+        self.create_object(
+            project, f"SELECT * INTO {project.test_schema}.mat_object FROM ({model_sql}) t"
+        )
+        run_dbt(["run"])
+
+
+class TestViewtoTable(BaseTableView):
+    """Test if changing from a view object to a table object correctly replaces"""
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"mat_object.sql": table_mat, "schema.yml": schema}
+
+    def test_passes(self, project):
+        self.create_object(project, f"CREATE VIEW {project.test_schema}.mat_object AS {model_sql}")
+        run_dbt(["run"])