feature/add-journey-models

.gitignore

@@ -74,3 +74,4 @@ venv.bak/
 
 # Claude instructions
 CLAUDE.md
+.claude/

.quickstart/quickstart.yml

@@ -12,6 +12,8 @@ table_variables:
     - user_unsubscribed_message_type
   iterable__using_event_extension:
     - event_extension
+  iterable__using_journey:
+    - journey
 
   _fivetran_quickstart_list_user:
     - list_user
@@ -31,6 +33,7 @@ destination_configurations:
 public_models: [
   "iterable__events",
   "iterable__campaigns",
+  "iterable__journeys",
   "iterable__user_unsubscriptions",
   "iterable__users",
   "iterable__user_campaign",
@@ -43,6 +46,11 @@ supported_vars:
     description: "Additional `EVENT_EXTENSION` fields to include in `iterable__events`"
     display_name: "Additional Event Extension Columns"
 
+  iterable_journey_pass_through_columns:
+    type: dictionary
+    description: "Additional `JOURNEY` fields to include in `iterable__journeys`"
+    display_name: "Additional Journey Columns"
+
   iterable_user_history_pass_through_columns:
     type: dictionary
     description: "Additional `USER_HISTORY` fields to include in `iterable__users` and `iterable__list_user_history`"

CHANGELOG.md

@@ -1,3 +1,18 @@
+# dbt_iterable v1.6.0
+
+[PR #84](https://github.com/fivetran/dbt_iterable/pull/84) includes the following updates:
+
+## Schema/Data Change
+**6 total changes • 0 possible breaking changes**
+
+| Data Model(s) | Change type | Old | New | Notes |
+| ------------- | ----------- | --- | --- | ----- |
+| `iterable__journeys` | New model | | | Journey-grain rollup of engagement metrics and unique users. |
+| `iterable__campaigns` | New fields | | `journey_id`, `journey_name`, `journey_type` | |
+| `int_iterable__journey_event_metrics` | New model | | | Aggregates campaign engagement metrics to the journey grain. |
+| `int_iterable__recurring_campaigns` | New field | | `journey_id` | |
+| `stg_iterable__journey`<br> `stg_iterable__journey_tmp`  | New models | | | Staging models for the new `journey` source table. | 
+
 # dbt_iterable v1.5.1
 
 [PR #83](https://github.com/fivetran/dbt_iterable/pull/83) includes the following updates:

README.md

@@ -36,6 +36,7 @@ By default, this package materializes the following final tables:
 | [iterable__events](https://fivetran.github.io/dbt_iterable/#!/model/model.iterable.iterable__events) | Tracks all user events with campaign attribution, user details, and channel information to analyze user behavior, conversion paths, and campaign effectiveness at the event level. See [tracked events details](https://fivetran.com/docs/applications/iterable#schemanotes). <br></br>**Example Analytics Questions:**<ul><li>What customer actions drive the most revenue and conversions?</li><li>Are our marketing campaigns more effective than organic customer discovery?</li><li>What customer journey paths lead to purchase or upgrade?</li></ul>|
 | [iterable__user_campaign](https://fivetran.github.io/dbt_iterable/#!/model/model.iterable.iterable__user_campaign) | Aggregates user-level engagement with specific campaigns and experiment variations including event counts by type to measure individual user responses to campaign messaging. <br></br>**Example Analytics Questions:**<ul><li>Which customers are most responsive to our campaigns, and what messages resonate with them?</li><li>Which email variations are winning with our audience?</li><li>How frequently do customers engage with each campaign?</li></ul>|
 | [iterable__campaigns](https://fivetran.github.io/dbt_iterable/#!/model/model.iterable.iterable__campaigns) | Tracks campaign performance with user interaction metrics, event counts, experiment variations, and template details to measure campaign effectiveness and optimize email strategy. <br></br>**Example Analytics Questions:**<ul><li>Which campaigns are driving the most engagement and sales?</li><li>How do different email designs and messaging strategies perform?</li><li>What's the reach and impact of our marketing initiatives?</li></ul>|
+| [iterable__journeys](https://fivetran.github.io/dbt_iterable/#!/model/model.iterable.iterable__journeys) | Tracks journey performance with engagement metrics, unique user counts, and can derive engagement rates rolled up from the campaigns belonging to each journey, so you can measure which automated sequences actually work. Journeys that send no messages, such as audience splits, are still included. <br></br>**Example Analytics Questions:**<ul><li>Which automated journeys are driving the most engagement?</li><li>How does our welcome series compare to our cart abandonment sequence?</li><li>Which journeys are enabled but no longer reaching anyone?</li></ul>|
 | [iterable__users](https://fivetran.github.io/dbt_iterable/#!/model/model.iterable.iterable__users) | Provides a comprehensive view of each user with campaign engagement history, list memberships, unsubscription status, and interaction metrics to understand user preferences and lifetime engagement. <br></br>**Example Analytics Questions:**<ul><li>Who are our most engaged customers and what do they have in common?</li><li>Why are customers unsubscribing, and what did we lose when they left?</li><li>Which acquisition channels bring us the most engaged customers?</li></ul>|
 | [iterable__list_user_history](https://fivetran.github.io/dbt_iterable/#!/model/model.iterable.iterable__list_user_history) | Chronicles user-list membership history to track when users join or leave lists, manage audience segmentation, and analyze list growth without excessive Monthly Active Rows (MAR) usage. <br></br>**Example Analytics Questions:**<ul><li>Are we growing or losing our subscriber base in key segments?</li><li>Which customers are moving between audience segments, and why?</li><li>How long do customers stay engaged before opting out?</li></ul>|
 | [iterable__user_unsubscriptions](https://fivetran.github.io/dbt_iterable/#!/model/model.iterable.iterable__user_unsubscriptions) | Tracks all user unsubscriptions by message type and channel to manage communication preferences, protect sender reputation, and identify unsubscribe patterns. <br></br>**Example Analytics Questions:**<ul><li>Which types of messages are causing customers to opt out?</li><li>Are we losing customers completely, or just from certain communication types?</li><li>What patterns indicate a customer is likely to unsubscribe?</li></ul>|
@@ -72,7 +73,7 @@ Include the following Iterable package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/iterable
-    version: [">=1.5.0", "<1.6.0"]
+    version: [">=1.6.0", "<1.7.0"]
 ```
 
 #### Database Incremental Strategies
@@ -133,15 +134,18 @@ vars:
     iterable__using_user_unsubscribed_message_type_history: false    # default is true
     iterable__using_campaign_suppression_list_history: false         # default is true   
     iterable__using_event_extension: false         # default is true   
+    iterable__using_journey: false                 # default is true
 ```
 
 ### (Optional) Additional configurations
 
 #### Passing Through Additional Fields
 
-This package includes fields we judged were standard across Iterable users. However, the Fivetran connector allows for additional columns to be brought through in the `event_extension` and `user_history` objects. Therefore, if you wish to bring them through, leverage our passthrough column variables. For `event_extension` columns, ensure that `iterable__using_event_extension` is set to True, which is the default.
+This package includes fields we judged were standard across Iterable users. However, the Fivetran connector allows for additional columns to be brought through in the `event_extension`, `journey`, and `user_history` objects. Therefore, if you wish to bring them through, leverage our passthrough column variables. For `event_extension` columns, ensure that `iterable__using_event_extension` is set to True, which is the default. The same applies to `iterable__using_journey` for `journey` columns.
 
-You will see these additional columns populate in the end `iterable__list_user_history`, `iterable__events`, and `iterable__users` models.
+You will see these additional columns populate in the end `iterable__list_user_history`, `iterable__events`, `iterable__journeys`, and `iterable__users` models.
+
+The `journey` object includes a set of `draft_*` fields whose exact names vary by connection, so they are not included by default. Use `iterable_journey_pass_through_columns` to bring through the specific draft fields you need.
 
 **Notice**: A `dbt run --full-refresh` is required each time these variables are edited.
 
@@ -155,6 +159,10 @@ vars:
     - name: "event_extension_field"
       alias: "renamed_field"
       transform_sql: "cast(renamed_field as string)"
+  iterable_journey_pass_through_columns:
+    - name: "draft_created_at"
+      alias: "draft_created_timestamp"
+    - name: "draft_name"
   iterable_user_history_pass_through_columns:
     - name: "user_attribute"
       alias: "renamed_user_attribute"

dbt_project.yml

@@ -1,6 +1,6 @@
 name: 'iterable'
 
-version: '1.5.1'
+version: '1.6.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<3.0.0"]
 models:
@@ -26,6 +26,7 @@ vars:
     channel: "{{ source('iterable', 'channel') }}"
     event: "{{ source('iterable', 'event') }}"
     event_extension: "{{ source('iterable', 'event_extension') }}"
+    journey: "{{ source('iterable', 'journey') }}"
     list: "{{ source('iterable', 'list') }}"
     list_user: "{{ source('iterable', 'list_user') }}"
     message_type: "{{ source('iterable', 'message_type') }}"
@@ -38,6 +39,7 @@ vars:
     user_unsubscribed_message_type_history: "{{ source('iterable', 'user_unsubscribed_message_type_history') }}"
 
     iterable_event_extension_pass_through_columns: []
+    iterable_journey_pass_through_columns: []
     iterable_user_history_pass_through_columns: []
 
     iterable__event_metrics:

integration_tests/ci/test_scenarios.yml

@@ -28,6 +28,7 @@ test_scenarios:
     vars:
       iterable_user_history_pass_through_columns: '[{"name": "phone_number_updated_at"}]'
       iterable_event_extension_pass_through_columns: '[{"name": "web_push_message"}]'
+      iterable_journey_pass_through_columns: '[{"name": "draft_name"}]'
     include_incremental: false
 
   - name: "features disabled"
@@ -36,6 +37,7 @@ test_scenarios:
       iterable__using_user_unsubscribed_message_type_history: false
       iterable__using_campaign_suppression_list_history: false
       iterable__using_event_extension: false
+      iterable__using_journey: false
     include_incremental: true
 
   - name: "MDLS"

integration_tests/dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'iterable_integration_tests'
-version: '1.5.1'
+version: '1.6.0'
 profile: 'integration_tests'
 vars:
   iterable:
@@ -12,6 +12,7 @@ vars:
     iterable_channel_identifier: "channel_data"
     iterable_event_identifier: "event_data"
     iterable_event_extension_identifier: "event_extension_data"
+    iterable_journey_identifier: "journey_data"
     iterable_list_identifier: "list_data"
     iterable_list_user_identifier: "list_user_data"
     iterable_message_type_identifier: "message_type_data"
@@ -59,6 +60,15 @@ seeds:
       +column_types:
         payload: "{{ 'varchar(100)'  if target.type in ('redshift', 'postgres') else 'string'}}"
         _fivetran_synced: timestamp
+    journey_data:
+      +column_types:
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        lifetime_limit: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        simultaneous_limit: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        start_tile_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        created_at: timestamp
+        updated_at: timestamp
+        _fivetran_synced: timestamp
     list_data:
       +column_types:
         _fivetran_synced: timestamp

integration_tests/seeds/journey_data.csv

@@ -0,0 +1,6 @@
+id,name,journey_type,description,creator_user_id,enabled,is_archived,lifetime_limit,simultaneous_limit,start_tile_id,trigger_event_names,created_at,updated_at,_fivetran_deleted,_fivetran_synced,draft_name
+157274,Welcome Series,Published,Onboarding sequence sent to new subscribers over their first week,jimmy.neutron@goddard.com,true,false,1,1,44001,"[""signUp""]",2021-06-01 09:14:22.000,2021-06-11 13:30:58.412,FALSE,2021-06-11 13:32:03.771,Welcome Series draft
+28801,Automation,Triggered,Behavioral automation triggered by in-app activity,jimmy.neutron@goddard.com,true,false,5,2,44002,"[""customEvent"",""emailOpen""]",2021-05-18 11:02:47.000,2021-06-09 17:45:10.220,FALSE,2021-06-11 13:32:03.771,Automation draft
+83250,Workflow - Survey,Blast,Post-purchase survey invitation,carl.wheezer@goddard.com,true,false,,1,44003,"[""purchase""]",2021-04-27 15:39:05.000,2021-06-02 08:21:36.905,FALSE,2021-06-11 13:32:03.771,
+900001,Audience Split - Dormant Users,Triggered,Splits dormant users into re-engagement branches and updates their profile fields. Sends no messages.,carl.wheezer@goddard.com,false,true,,,44004,[],2021-03-15 10:00:00.000,2021-05-20 12:11:44.310,FALSE,2021-06-11 13:32:03.771,
+900002,Deleted Journey,Triggered,Journey deleted in the source and excluded from staging output,carl.wheezer@goddard.com,false,false,,,44005,"[""signUp""]",2021-02-08 08:45:12.000,2021-04-02 16:20:31.118,TRUE,2021-06-11 13:32:03.771,

integration_tests/tests/consistency/consistency_journeys.sql

@@ -0,0 +1,51 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+{% set exclude_cols = ['trigger_event_names'] + var('consistency_test_exclude_metrics', []) %}
+{% set fields = dbt_utils.star(from=ref('iterable__journeys'), except=exclude_cols) %}
+
+-- this test ensures the iterable__journeys end model matches the prior version
+with prod as (
+    select {{ fields }}
+    from {{ target.schema }}_iterable_prod.iterable__journeys
+    where date(updated_at) < date({{ dbt.current_timestamp() }})
+),
+
+dev as (
+    select {{ fields }}
+    from {{ target.schema }}_iterable_dev.iterable__journeys
+    where date(updated_at) < date({{ dbt.current_timestamp() }})
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

macros/staging/get_journey_columns.sql

@@ -0,0 +1,25 @@
+{% macro get_journey_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "creator_user_id", "datatype": dbt.type_string()},
+    {"name": "description", "datatype": dbt.type_string()},
+    {"name": "enabled", "datatype": "boolean"},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "is_archived", "datatype": "boolean"},
+    {"name": "journey_type", "datatype": dbt.type_string()},
+    {"name": "lifetime_limit", "datatype": dbt.type_int()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "simultaneous_limit", "datatype": dbt.type_int()},
+    {"name": "start_tile_id", "datatype": dbt.type_int()},
+    {"name": "trigger_event_names", "datatype": dbt.type_string()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('iterable_journey_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/intermediate/int_iterable.yml

@@ -11,6 +11,9 @@ models:
   - name: int_iterable__campaign_lists
     description: Table that identifies the suppressed and send lists associated with a campaign.
 
+  - name: int_iterable__journey_event_metrics
+    description: Table that generates relevant total and unique journey level aggregations, rolled up from the user/campaign grain via the campaign's journey reference.
+
   - name: int_iterable__latest_campaign
     description: Table that filters out historical versions of the campaign, and returns the most recent campaign version.
 

models/intermediate/int_iterable__journey_event_metrics.sql

@@ -0,0 +1,48 @@
+{{ config(enabled=var('iterable__using_journey', True)) }}
+
+with user_campaign as (
+    select *
+    from {{ ref('iterable__user_campaign') }}
+
+), campaign as (
+
+    select
+        source_relation,
+        campaign_id,
+        journey_id
+
+    from {{ ref('int_iterable__recurring_campaigns') }}
+    where journey_id is not null
+
+), journey_event_metrics as (
+{%- set user_campaign_columns = adapter.get_columns_in_relation(ref('iterable__user_campaign')) %}
+{%- set non_agg_columns = ['source_relation', 'unique_user_key', 'user_id', '_fivetran_user_id',
+    'user_email', 'user_full_name', 'campaign_id', 'campaign_name', 'recurring_campaign_id',
+    'recurring_campaign_name', 'first_event_at', 'last_event_at', 'template_id', 'template_name',
+    'experiment_id', 'unique_user_campaign_id', 'first_open_or_click_event_at'
+    ] %}
+
+    select
+        campaign.source_relation,
+        campaign.journey_id,
+        count(distinct user_campaign.campaign_id) as count_total_campaigns,
+        count(distinct user_campaign.unique_user_key) as count_unique_users,
+        min(user_campaign.first_event_at) as first_event_at,
+        max(user_campaign.last_event_at) as last_event_at
+        {% for col in user_campaign_columns if col.name|lower not in non_agg_columns %}
+            , sum(user_campaign.{{ col.name }}) as {{ col.name }}
+            -- counts distinct users; campaign_event_metrics counts distinct emails
+            , count(distinct case when user_campaign.{{ col.name }} > 0 then user_campaign.unique_user_key else null end) as unique_{{ col.name }}
+        {% endfor %}
+
+    from user_campaign
+
+    inner join campaign
+        on user_campaign.campaign_id = campaign.campaign_id
+        and user_campaign.source_relation = campaign.source_relation
+
+    {{ dbt_utils.group_by(n=2) }}
+)
+
+select *
+from journey_event_metrics

models/intermediate/int_iterable__recurring_campaigns.sql

@@ -18,6 +18,11 @@ with latest_campaign as (
         latest_campaign.created_by_user_id,
         latest_campaign.template_id,
         latest_campaign.recurring_campaign_id,
+        {% if var('iterable__using_journey', true) %}
+        -- cast to string so this matches journey_id in stg_iterable__journey; workflow_id arrives
+        -- numeric from campaign_history but as a string from event_extension
+        cast(latest_campaign.workflow_id as {{ dbt.type_string() }}) as journey_id,
+        {% endif %}
 
         recurring_campaign.campaign_name as recurring_campaign_name,
         recurring_campaign.campaign_state as recurring_campaign_state,