APP-1316: add BigQuery expiration to elementary test temp tables

macros/edr/tests/test_utils/create_elementary_test_table.sql

@@ -31,8 +31,11 @@
             type="table",
         ) -%}
 
-        {# Create the table if it doesnt exist #}
-        {%- do elementary.create_or_replace(false, temp_table_relation, sql_query) %}
+        {# Table must be non-temporary so it's visible across dbt sessions (e.g. on_run_end cleanup).
+           expiration_hours is a safety net for adapters that support it (currently BigQuery). #}
+        {%- do elementary.create_or_replace(
+            false, temp_table_relation, sql_query, expiration_hours=1
+        ) %}
 
         {# Cache the test table for easy access later #}
         {% set test_entry = elementary.get_cache(

macros/utils/table_operations/create_or_replace.sql

@@ -1,64 +1,120 @@
-{% macro create_or_replace(temporary, relation, sql_query) %}
+{% macro create_or_replace(temporary, relation, sql_query, expiration_hours=none) %}
     {{
         return(
             adapter.dispatch("create_or_replace", "elementary")(
-                temporary, relation, sql_query
+                temporary, relation, sql_query, expiration_hours=expiration_hours
             )
         )
     }}
 {% endmacro %}
 
 {# Snowflake and Bigquery #}
-{% macro default__create_or_replace(temporary, relation, sql_query) %}
-    {% do elementary.edr_create_table_as(temporary, relation, sql_query) %}
+{% macro default__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
+    {% do elementary.edr_create_table_as(
+        temporary, relation, sql_query, expiration_hours=expiration_hours
+    ) %}
 {% endmacro %}
 
-{% macro redshift__create_or_replace(temporary, relation, sql_query) %}
+{% macro redshift__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% do elementary.edr_create_table_as(
-        temporary, relation, sql_query, drop_first=true, should_commit=true
+        temporary,
+        relation,
+        sql_query,
+        drop_first=true,
+        should_commit=true,
+        expiration_hours=expiration_hours,
     ) %}
 {% endmacro %}
 
-{% macro postgres__create_or_replace(temporary, relation, sql_query) %}
+{% macro postgres__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% do elementary.run_query("BEGIN") %}
     {% do elementary.edr_create_table_as(
-        temporary, relation, sql_query, drop_first=true
+        temporary,
+        relation,
+        sql_query,
+        drop_first=true,
+        expiration_hours=expiration_hours,
     ) %}
     {% do elementary.run_query("COMMIT") %}
 {% endmacro %}
 
-{% macro spark__create_or_replace(temporary, relation, sql_query) %}
+{% macro spark__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% do elementary.edr_create_table_as(
-        temporary, relation, sql_query, drop_first=true, should_commit=true
+        temporary,
+        relation,
+        sql_query,
+        drop_first=true,
+        should_commit=true,
+        expiration_hours=expiration_hours,
     ) %}
 {% endmacro %}
 
-{% macro fabricspark__create_or_replace(temporary, relation, sql_query) %}
-    {{ return(elementary.spark__create_or_replace(temporary, relation, sql_query)) }}
+{% macro fabricspark__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
+    {{
+        return(
+            elementary.spark__create_or_replace(
+                temporary, relation, sql_query, expiration_hours=expiration_hours
+            )
+        )
+    }}
 {% endmacro %}
 
-{% macro athena__create_or_replace(temporary, relation, sql_query) %}
+{% macro athena__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% do elementary.edr_create_table_as(
-        temporary, relation, sql_query, drop_first=true
+        temporary,
+        relation,
+        sql_query,
+        drop_first=true,
+        expiration_hours=expiration_hours,
     ) %}
 {% endmacro %}
 
-{% macro trino__create_or_replace(temporary, relation, sql_query) %}
+{% macro trino__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% do elementary.edr_create_table_as(
-        temporary, relation, sql_query, drop_first=true
+        temporary,
+        relation,
+        sql_query,
+        drop_first=true,
+        expiration_hours=expiration_hours,
     ) %}
 {% endmacro %}
 
-{% macro clickhouse__create_or_replace(temporary, relation, sql_query) %}
+{% macro clickhouse__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% do elementary.edr_create_table_as(
-        temporary, relation, sql_query, drop_first=true
+        temporary,
+        relation,
+        sql_query,
+        drop_first=true,
+        expiration_hours=expiration_hours,
     ) %}
 {% endmacro %}
 
 {# DuckDB uses CREATE OR REPLACE TABLE, so drop_first is not needed.
    should_commit=true ensures the table survives the ROLLBACK issued by test connections. #}
-{% macro duckdb__create_or_replace(temporary, relation, sql_query) %}
+{% macro duckdb__create_or_replace(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% do elementary.edr_create_table_as(
-        temporary, relation, sql_query, should_commit=true
+        temporary,
+        relation,
+        sql_query,
+        should_commit=true,
+        expiration_hours=expiration_hours,
     ) %}
 {% endmacro %}

macros/utils/table_operations/create_table_as.sql

@@ -1,5 +1,10 @@
 {% macro edr_create_table_as(
-    temporary, relation, sql_query, drop_first=false, should_commit=false
+    temporary,
+    relation,
+    sql_query,
+    drop_first=false,
+    should_commit=false,
+    expiration_hours=none
 ) %}
     {# This macro contains a simplified implementation that replaces our usage of 
      dbt.create_table_as and serves our needs.
@@ -8,49 +13,63 @@
     {% if drop_first %} {% do dbt.drop_relation_if_exists(relation) %} {% endif %}
 
     {% set create_query = elementary.edr_get_create_table_as_sql(
-        temporary, relation, sql_query
+        temporary, relation, sql_query, expiration_hours=expiration_hours
     ) %}
     {% do elementary.run_query(create_query) %}
 
     {% if should_commit %} {% do adapter.commit() %} {% endif %}
 {% endmacro %}
 
 
-{% macro edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {{
         return(
             adapter.dispatch("edr_get_create_table_as_sql", "elementary")(
-                temporary, relation, sql_query
+                temporary, relation, sql_query, expiration_hours=expiration_hours
             )
         )
     }}
 {% endmacro %}
 
-{% macro default__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro default__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {{ dbt.get_create_table_as_sql(temporary, relation, sql_query) }}
 {% endmacro %}
 
 {# Simplified versions for dbt-fusion supported adapters as the original dbt macro 
    no longer works outside of the scope of a model's materialization #}
-{% macro snowflake__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro snowflake__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
   create or replace {% if temporary %} temporary {% endif %} table {{ relation }}
   as {{ sql_query }}
 {% endmacro %}
 
-{% macro bigquery__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro bigquery__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
   create or replace table {{ relation }}
     {% if temporary %}
   options (expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 hour))
+    {% elif expiration_hours is not none %}
+  options (expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL {{ expiration_hours }} hour))
     {% endif %}
   as {{ sql_query }}
 {% endmacro %}
 
-{% macro postgres__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro postgres__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
   create {% if temporary %} temporary {% endif %} table {{ relation.include(database=(not temporary), schema=(not temporary)) }}
   as {{ sql_query }}
 {% endmacro %}
 
-{% macro redshift__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro redshift__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% if temporary and elementary.is_dbt_fusion() %}
         {# dbt-fusion uses connection pooling - temp tables created in one session
        aren't visible in other sessions. Create regular tables instead.
@@ -63,7 +82,9 @@
     {% endif %}
 {% endmacro %}
 
-{% macro databricks__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro databricks__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {% if temporary %}
         {% if elementary.is_dbt_fusion() %}
             {# 
@@ -83,24 +104,32 @@
   as {{ sql_query }}
 {% endmacro %}
 
-{% macro clickhouse__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro clickhouse__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {# ClickHouse does not support database-scoped temporary tables, so we force temporary to be false. #}
     {{ dbt.get_create_table_as_sql(false, relation, sql_query) }}
 {% endmacro %}
 
-{% macro duckdb__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro duckdb__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
   create or replace {% if temporary %} temporary {% endif %} table {{ relation }}
   as {{ sql_query }}
 {% endmacro %}
 
-{% macro trino__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro trino__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {# dbt-trino's create_table_as accesses model.config which fails when called
      outside a model context (e.g. from edr_create_table_as). Use simplified SQL. #}
   create table {{ relation }}
   as {{ sql_query }}
 {% endmacro %}
 
-{% macro spark__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro spark__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {# Spark: use a temporary view for temp tables, regular table otherwise #}
     {% if temporary %}
     create or replace temporary view {{ relation }}
@@ -111,11 +140,13 @@
     {% endif %}
 {% endmacro %}
 
-{% macro fabricspark__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+{% macro fabricspark__edr_get_create_table_as_sql(
+    temporary, relation, sql_query, expiration_hours=none
+) %}
     {{
         return(
             elementary.spark__edr_get_create_table_as_sql(
-                temporary, relation, sql_query
+                temporary, relation, sql_query, expiration_hours=expiration_hours
             )
         )
     }}