diff --git a/quickstart/101-chaos-studio-workspace-azapi/main.tf b/quickstart/101-chaos-studio-workspace-azapi/main.tf new file mode 100644 index 000000000..8e9372276 --- /dev/null +++ b/quickstart/101-chaos-studio-workspace-azapi/main.tf @@ -0,0 +1,122 @@ +# Chaos Studio v2 - Quickstart 101: a minimal AzAPI workspace +# ============================================================================= +# This quickstart provisions the smallest useful Chaos Studio v2 footprint: +# +# * a resource group (with a random, human-readable name suffix), +# * a user-assigned managed identity for the workspace, +# * a Reader role assignment so that identity can discover in-scope resources, +# * a Microsoft.Chaos/workspaces resource, scoped to the resource group. +# +# It deliberately stops there: it does NOT author scenarios, scenario +# configurations, or run any fault actions. It is safe to `terraform apply` and +# `terraform destroy` repeatedly - nothing here injects a fault. Its job is to +# teach the AzAPI type/api-version idiom and the prerequisite identity + RBAC +# wiring that every Chaos workspace needs. +# +# Why AzAPI and not the azurerm provider? Microsoft.Chaos v2 is a *preview* +# api-version. The azurerm provider has no typed resource for it (and cannot, +# on the preview timeline), so we express the workspace as a raw ARM resource +# through the AzAPI provider, which can target any type + api-version. See +# https://learn.microsoft.com/en-us/azure/developer/terraform/overview-azapi-provider. +# +# For the full end-to-end workflow - a real fault target, a scenario +# configuration, and the discover -> evaluate -> validate -> fixResourcePermissions +# -> execute chain - see the companion quickstart +# `201-chaos-studio-scenario-run-azapi`. +# ============================================================================= + +# BEGIN chaos-codegen:api-versions +# Machine-managed Microsoft.Chaos api-version pins (api-version 2026-08-01-preview). +# Do NOT edit inside these markers by hand - they are regenerated from the +# Microsoft.Chaos ARM specification. Only this locals block is machine-managed; +# every other block in this file is hand-authored and never touched. +locals { + chaos = { + workspace = "Microsoft.Chaos/workspaces@2026-08-01-preview" + scenario = "Microsoft.Chaos/workspaces/scenarios@2026-08-01-preview" + scenario_configuration = "Microsoft.Chaos/workspaces/scenarios/configurations@2026-08-01-preview" + scenario_run = "Microsoft.Chaos/workspaces/scenarios/runs@2026-08-01-preview" + discovered_resource = "Microsoft.Chaos/workspaces/discoveredResources@2026-08-01-preview" + latest_validation = "Microsoft.Chaos/workspaces/scenarios/configurations/validations@2026-08-01-preview" + latest_permission_fix = "Microsoft.Chaos/workspaces/scenarios/configurations/fixResourcePermissions@2026-08-01-preview" + } +} +# END chaos-codegen:api-versions + +# The subscription the deployment runs in, used to build the built-in role +# definition resource id below. +data "azurerm_subscription" "current" {} + +# A random, readable suffix so repeated applies in the same subscription do not +# collide on resource names. +resource "random_pet" "this" { + length = 2 + separator = "-" +} + +locals { + resource_group_name = "${var.name_prefix}-${random_pet.this.id}" + + # Reader (built-in role). Chosen over Owner/Contributor: discovery only needs + # read access to enumerate in-scope resources; per-scenario action roles are + # granted separately by the fixResourcePermissions action (see the 201 + # quickstart), never by a broad standing grant here. + # + # GUID verified against the Azure built-in role catalog: + # https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/general#reader + reader_role_definition_id = "/subscriptions/${data.azurerm_subscription.current.subscription_id}/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7" +} + +resource "azurerm_resource_group" "this" { + name = local.resource_group_name + location = var.location + tags = var.tags +} + +# User-assigned (not system-assigned) managed identity for the workspace. Using +# a user-assigned identity lets the Reader assignment below be created and +# destroyed on its own lifecycle, independently of the workspace resource. +resource "azurerm_user_assigned_identity" "chaos" { + name = "${local.resource_group_name}-id" + resource_group_name = azurerm_resource_group.this.name + location = azurerm_resource_group.this.location + tags = var.tags +} + +# Grant the workspace identity Reader over the scope the workspace targets, so a +# later discovery can enumerate the in-scope resources. +resource "azurerm_role_assignment" "reader" { + scope = azurerm_resource_group.this.id + role_definition_id = local.reader_role_definition_id + principal_id = azurerm_user_assigned_identity.chaos.principal_id +} + +# The Chaos Studio workspace, expressed as a raw ARM resource via AzAPI. `type` +# references the codegen-owned pin in local.chaos (above), so a spec/api-version +# bump is a one-line regeneration, never a hand edit of this block. +resource "azapi_resource" "workspace" { + type = local.chaos.workspace + name = "${random_pet.this.id}-chaos" + parent_id = azurerm_resource_group.this.id + location = azurerm_resource_group.this.location + tags = var.tags + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.chaos.id] + } + + body = { + properties = { + # `scopes` is the only spec-required workspace property: the ARM ids the + # workspace's child scenarios are allowed to target. Here it is the + # resource group created above. + scopes = [azurerm_resource_group.this.id] + } + } + + # The Reader assignment must exist before the workspace so the identity is + # usable for discovery. Explicit because the dependency flows through + # principal_id, which Terraform cannot infer as a resource-id reference. + depends_on = [azurerm_role_assignment.reader] +} diff --git a/quickstart/101-chaos-studio-workspace-azapi/outputs.tf b/quickstart/101-chaos-studio-workspace-azapi/outputs.tf new file mode 100644 index 000000000..dc56f1478 --- /dev/null +++ b/quickstart/101-chaos-studio-workspace-azapi/outputs.tf @@ -0,0 +1,14 @@ +output "workspace_id" { + description = "The fully-qualified ARM id of the Chaos Studio workspace." + value = azapi_resource.workspace.id +} + +output "workspace_identity_principal_id" { + description = "The principal id of the user-assigned identity the workspace uses for discovery. Grant this identity any additional roles a scenario requires." + value = azurerm_user_assigned_identity.chaos.principal_id +} + +output "resource_group_name" { + description = "The resource group that holds the workspace and is its default scope." + value = azurerm_resource_group.this.name +} diff --git a/quickstart/101-chaos-studio-workspace-azapi/providers.tf b/quickstart/101-chaos-studio-workspace-azapi/providers.tf new file mode 100644 index 000000000..809a9930e --- /dev/null +++ b/quickstart/101-chaos-studio-workspace-azapi/providers.tf @@ -0,0 +1,33 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + azapi = { + source = "Azure/azapi" + # >= 2.11.0: earlier azapi versions set azapi_resource_action output + # incorrectly while an action was still executing + # (Azure/terraform-provider-azapi GH-1168, fixed in 2.11.0). The 101 uses + # no actions, but both quickstarts pin the provider identically so the + # cross-example contract stays uniform. + version = ">= 2.11.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} + +provider "azapi" {} + +provider "azurerm" { + # azurerm v4 requires an explicit subscription id. When var.subscription_id is + # null the provider falls back to the ARM_SUBSCRIPTION_ID environment variable, + # so `az account set` / CI-provided credentials work without editing this file. + subscription_id = var.subscription_id + features {} +} diff --git a/quickstart/101-chaos-studio-workspace-azapi/readme.md b/quickstart/101-chaos-studio-workspace-azapi/readme.md new file mode 100644 index 000000000..6b605a02f --- /dev/null +++ b/quickstart/101-chaos-studio-workspace-azapi/readme.md @@ -0,0 +1,100 @@ +# Quickstart 101 - Chaos Studio v2 workspace (AzAPI, minimal) + +Provision the smallest useful Azure Chaos Studio v2 footprint with Terraform and +the [AzAPI provider](https://learn.microsoft.com/en-us/azure/developer/terraform/overview-azapi-provider): +a resource group, a user-assigned managed identity, a `Reader` role assignment, +and a `Microsoft.Chaos/workspaces` resource scoped to the resource group. + +This example is intentionally minimal. It does **not** create scenarios, scenario +configurations, or run any fault actions, so it is safe to `apply` and `destroy` +repeatedly - nothing here injects a fault. For the full end-to-end workflow (a +real target, a scenario configuration, and the +`discover -> evaluate -> validate -> fixResourcePermissions -> execute` chain), +see the companion quickstart +[`201-chaos-studio-scenario-run-azapi`](../201-chaos-studio-scenario-run-azapi). + +## Why AzAPI instead of the azurerm provider? + +`Microsoft.Chaos` v2 is a **preview** api-version (`2026-08-01-preview`). The +`azurerm` provider has no typed resource for it and cannot offer one on the +preview timeline, so the workspace is expressed as a raw ARM resource through the +generic AzAPI provider, which can target any resource type and api-version. This +is the documented pattern for preview Azure services. Native `azurerm` support is +being delivered separately and is additive - these AzAPI examples remain the way +to reach preview api-versions. + +> **Preview caveat.** Because this targets a preview api-version, the resource +> shape and behavior can change between preview releases, and the feature must be +> enabled for your subscription and available in your chosen region. Pin the +> api-version (this example does, via the codegen-owned `local.chaos` block in +> `main.tf`) and re-validate after any spec bump. + +## Prerequisites + +- Terraform `>= 1.5`. +- An Azure subscription with the Chaos Studio v2 preview enabled, and a region + where it is available (set `location`). +- **A target subscription.** Set `ARM_SUBSCRIPTION_ID` (the value `az account set` + already exports) or pass `-var 'subscription_id='`. The azurerm v4 provider + requires a subscription id; the example leaves `subscription_id` null so it is + sourced from the environment by default. +- **Permission to create role assignments on the target scope.** This example + assigns `Reader` to the workspace identity, which requires the deploying + principal to hold a role with `Microsoft.Authorization/roleAssignments/write` + on the scope - for example **Owner** or **User Access Administrator** on the + subscription or resource group. A principal with only Contributor cannot create + the role assignment and the apply will fail. + +## Usage + +```bash +export ARM_SUBSCRIPTION_ID="$(az account show --query id -o tsv)" +terraform init +terraform apply +``` + +Override the defaults as needed: + +```bash +terraform apply -var 'location=westus2' -var 'name_prefix=demo-chaos' -var 'subscription_id=00000000-0000-0000-0000-000000000000' +``` + +Outputs: + +| Output | Description | +|--------|-------------| +| `workspace_id` | ARM id of the created Chaos Studio workspace. | +| `workspace_identity_principal_id` | Principal id of the workspace's user-assigned identity; grant it any additional roles a scenario needs. | +| `resource_group_name` | The resource group holding the workspace (its default scope). | + +## AzAPI action lifecycle caveats + +This minimal quickstart runs no actions, but the same AzAPI behaviors apply to +the `201` quickstart and to any workflow you build on top of this one, so they +are called out here as well: + +- **Actions fire on create/replace only.** An `azapi_resource_action` runs when + it is first created (or replaced); it does **not** re-fire on a subsequent + unchanged `terraform apply`. To re-run an action, taint or replace it + (`terraform apply -replace=
`). +- **`terraform destroy` performs no action** unless the action block sets + `when = "destroy"`. Destroying does not automatically undo an action's + server-side effect. +- **`fixResourcePermissions` with `whatIf = false` creates real role + assignments** that Terraform does not track in state and will not remove on + destroy. Use `whatIf = true` to preview the required assignments first, and + clean up any created assignments out of band. + +## What this creates + +| Resource | Purpose | +|----------|---------| +| `random_pet.this` | Readable, unique name suffix. | +| `azurerm_resource_group.this` | Holds the workspace; also its default scope. | +| `azurerm_user_assigned_identity.chaos` | Identity the workspace uses for discovery. | +| `azurerm_role_assignment.reader` | Grants that identity `Reader` on the scope. | +| `azapi_resource.workspace` | The `Microsoft.Chaos/workspaces` resource. | + +The `Reader` role definition GUID (`acdd72a7-3385-48ef-bd42-f606fba81ae7`) is +verified against the +[Azure built-in role catalog](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/general#reader). diff --git a/quickstart/101-chaos-studio-workspace-azapi/variables.tf b/quickstart/101-chaos-studio-workspace-azapi/variables.tf new file mode 100644 index 000000000..a58d2c1ef --- /dev/null +++ b/quickstart/101-chaos-studio-workspace-azapi/variables.tf @@ -0,0 +1,28 @@ +variable "subscription_id" { + description = "Azure subscription ID to deploy into. The azurerm v4 provider requires a subscription id; when this is left null it is sourced from the ARM_SUBSCRIPTION_ID environment variable (which `az account set` and most CI set automatically). Set it explicitly to target a specific subscription. (The Reader role-definition id in main.tf is derived separately from the data.azurerm_subscription.current data source, not from this variable.)" + type = string + default = null +} + +variable "location" { + description = "Azure region for the resource group and workspace. Must be a region where the Microsoft.Chaos v2 preview is enabled." + type = string + default = "eastus" +} + +variable "name_prefix" { + description = "Prefix for generated resource names. A random pet suffix is appended for uniqueness." + type = string + default = "chaos-quickstart" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,20}$", var.name_prefix)) + error_message = "name_prefix must be 2-21 characters: lowercase letters, digits, or hyphens, starting with a letter." + } +} + +variable "tags" { + description = "Tags applied to every resource this quickstart creates." + type = map(string) + default = {} +} diff --git a/quickstart/201-chaos-studio-scenario-run-azapi/main.tf b/quickstart/201-chaos-studio-scenario-run-azapi/main.tf new file mode 100644 index 000000000..249cec24d --- /dev/null +++ b/quickstart/201-chaos-studio-scenario-run-azapi/main.tf @@ -0,0 +1,544 @@ +# Chaos Studio v2 - Quickstart 201: an end-to-end AzAPI scenario run +# ============================================================================= +# This quickstart provisions a realistic, end-to-end Chaos Studio v2 drill: +# +# * a resource group, a virtual network + subnet, and a network security group +# associated with that subnet - the fault target, +# * a user-assigned managed identity the workspace uses for discovery, +# * a Reader role assignment so that identity can discover in-scope resources, +# * a Microsoft.Chaos/workspaces resource scoped to the resource group, +# * a ScenarioConfiguration for a built-in scenario (default: DnsOutage-1.0) +# whose targeting AND parameters come from variables, so naming a different +# built-in scenario is a real substitution rather than a cosmetic one, and +# * the full action chain, sequenced with depends_on: +# discover -> evaluate -> validate -> fixResourcePermissions -> execute +# +# The `execute` action - the only block that injects real faults - is gated +# behind var.execute_scenario (default false). So the default `terraform apply` +# provisions, discovers, evaluates, validates, and PREVIEWS the permission fix +# (whatIf) WITHOUT injecting a fault, and is safe to apply/destroy repeatedly. +# When var.execute_scenario is true the permission fix becomes a REAL remediation +# (whatIf = false), because a preview creates no role assignments and the run +# would then fail on missing permissions - see local.fix_permissions_what_if and +# var.fix_permissions_what_if. A `when = destroy` +# cancel action makes a best-effort cancel of a run that is still active at +# destroy time after a *successful* apply; it cannot rescue an interrupted apply +# (see the cancel_run block and readme.md for the limitation and the manual +# remediation you must run in that case). +# +# Why AzAPI and not the azurerm provider? Microsoft.Chaos v2 is a *preview* +# api-version; the azurerm provider has no typed resource for it, so the Chaos +# resources are expressed as raw ARM through the AzAPI provider. See +# https://learn.microsoft.com/en-us/azure/developer/terraform/overview-azapi-provider. +# +# For the minimal getting-started example (just a workspace), see the companion +# quickstart `101-chaos-studio-workspace-azapi`. +# ============================================================================= + +# BEGIN chaos-codegen:api-versions +# Machine-managed Microsoft.Chaos api-version pins (api-version 2026-08-01-preview). +# Do NOT edit inside these markers by hand - they are regenerated from the +# Microsoft.Chaos ARM specification. Only this locals block is machine-managed; +# every other block in this file is hand-authored and never touched. +locals { + chaos = { + workspace = "Microsoft.Chaos/workspaces@2026-08-01-preview" + scenario = "Microsoft.Chaos/workspaces/scenarios@2026-08-01-preview" + scenario_configuration = "Microsoft.Chaos/workspaces/scenarios/configurations@2026-08-01-preview" + scenario_run = "Microsoft.Chaos/workspaces/scenarios/runs@2026-08-01-preview" + discovered_resource = "Microsoft.Chaos/workspaces/discoveredResources@2026-08-01-preview" + latest_validation = "Microsoft.Chaos/workspaces/scenarios/configurations/validations@2026-08-01-preview" + latest_permission_fix = "Microsoft.Chaos/workspaces/scenarios/configurations/fixResourcePermissions@2026-08-01-preview" + } +} +# END chaos-codegen:api-versions + +# The subscription the deployment runs in, used to build the built-in role +# definition resource id below. +data "azurerm_subscription" "current" {} + +# A random, readable suffix so repeated applies in the same subscription do not +# collide on resource names. +resource "random_pet" "this" { + length = 2 + separator = "-" +} + +locals { + resource_group_name = "${var.name_prefix}-${random_pet.this.id}" + + # Reader (built-in role). Discovery only needs read access to enumerate + # in-scope resources; scenario-specific action roles are granted by the + # fixResourcePermissions action below, never by a broad standing grant. + # GUID verified against the Azure built-in role catalog: + # https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/general#reader + reader_role_definition_id = "/subscriptions/${data.azurerm_subscription.current.subscription_id}/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7" + + # The built-in scenario is referenced as a child of the workspace. The + # ScenarioConfiguration lives under this scenario and points back at it via + # body.properties.scenarioId. + scenario_id = "${azapi_resource.workspace.id}/scenarios/${var.scenario_name}" + + # Which discovered resources the run acts on. Empty (the default) means "the + # fault target this example creates", so a plain `terraform apply` resolves + # something without the reader supplying an id; set var.target_resource_ids to + # point the same scenario at resources you already own. + target_resource_ids = length(var.target_resource_ids) > 0 ? var.target_resource_ids : [azurerm_network_security_group.target.id] + + # whatIf flag for the fixResourcePermissions action. A dry run (true) reports + # the assignments the workspace identity is missing but creates NONE of them, so + # an execute that follows one would fail on missing permissions. Hence the + # default (var.fix_permissions_what_if = null) DERIVES the flag from + # var.execute_scenario: preview when not executing, real remediation when + # executing. Setting the variable explicitly overrides that, and the + # precondition on the action below rejects the one incoherent combination + # (an explicit dry run while executing). + fix_permissions_what_if = var.fix_permissions_what_if == null ? !var.execute_scenario : var.fix_permissions_what_if +} + +resource "azurerm_resource_group" "this" { + name = local.resource_group_name + location = var.location + tags = var.tags +} + +resource "azurerm_virtual_network" "this" { + name = "${local.resource_group_name}-vnet" + resource_group_name = azurerm_resource_group.this.name + location = azurerm_resource_group.this.location + address_space = ["10.0.0.0/16"] + tags = var.tags +} + +resource "azurerm_subnet" "this" { + name = "internal" + resource_group_name = azurerm_resource_group.this.name + virtual_network_name = azurerm_virtual_network.this.name + address_prefixes = ["10.0.1.0/24"] +} + +# User-assigned (not system-assigned) managed identity for the workspace, so the +# Reader assignment has an independent lifecycle from the workspace resource. +resource "azurerm_user_assigned_identity" "chaos" { + name = "${local.resource_group_name}-id" + resource_group_name = azurerm_resource_group.this.name + location = azurerm_resource_group.this.location + tags = var.tags +} + +# Grant the workspace identity Reader over the scope the workspace targets, so a +# later discovery can enumerate the in-scope resources. +resource "azurerm_role_assignment" "reader" { + scope = azurerm_resource_group.this.id + role_definition_id = local.reader_role_definition_id + principal_id = azurerm_user_assigned_identity.chaos.principal_id +} + +# The fault target: a network security group, chosen because the default +# scenario (DnsOutage-1.0) is SERVICE-DIRECT. Chaos Studio v2 built-in templates +# come in two flavours: +# +# * service-direct - the fault acts on the Azure resource through its own +# control plane (here: temporarily adding deny rules to this NSG). No agent, +# no VM, and no managed identity on the target. +# * agent-based - the CPU / memory pressure family, where Chaos Studio +# installs the Chaos Agent VM extension on the target's instances and the +# agent authenticates as the TARGET's own managed identity. +# +# A service-direct target keeps the example cheap (an NSG costs nothing), fast, +# and - decisively - actually runnable: agent faults against +# Microsoft.Compute/virtualMachineScaleSets are turned OFF in production, so a +# scale-set + agent pairing can never execute against a production subscription. +# See readme.md ("Choosing a target") before substituting your own target. +# +# No security_rule blocks are declared on purpose: the rules the fault adds and +# reverts are service-managed, and leaving the attribute unset keeps Terraform +# from trying to reconcile them mid-run. +resource "azurerm_network_security_group" "target" { + name = "${local.resource_group_name}-nsg" + resource_group_name = azurerm_resource_group.this.name + location = azurerm_resource_group.this.location + tags = var.tags +} + +# Associate the NSG with the subnet so the deny rules the fault applies have a +# real blast radius rather than acting on an unattached resource. +resource "azurerm_subnet_network_security_group_association" "target" { + subnet_id = azurerm_subnet.this.id + network_security_group_id = azurerm_network_security_group.target.id +} + +# The Chaos Studio workspace, scoped to the resource group that holds the target. +resource "azapi_resource" "workspace" { + type = local.chaos.workspace + name = "${random_pet.this.id}-chaos" + parent_id = azurerm_resource_group.this.id + location = azurerm_resource_group.this.location + tags = var.tags + + # The one identity the whole chain depends on: discovery enumerates as it, + # fixResourcePermissions grants the scenario's roles TO it, and the run's + # service-direct faults act as it. + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.chaos.id] + } + + body = { + properties = { + scopes = [azurerm_resource_group.this.id] + } + } + + # The Reader assignment must exist before the workspace so the identity is + # usable for discovery. Explicit because the dependency flows through + # principal_id, which Terraform cannot infer as a resource-id reference. + depends_on = [azurerm_role_assignment.reader] +} + +# discover: enumerate the in-scope resources. Acts on discoveries/latest, returns +# 202 + Location; the AzAPI provider polls the LRO to completion. +resource "azapi_resource_action" "discover" { + type = local.chaos.workspace + resource_id = azapi_resource.workspace.id + action = "discover" + + # Discovery snapshots whatever is in scope AT THE MOMENT IT RUNS, so the fault + # target must already exist. None of this block's arguments reference the NSG, so + # Terraform infers no edge and would otherwise be free to create the NSG + # concurrently with - or after - this action; the discovery would then return a + # resource group without the target, and the validate / execute actions further + # down the chain would fail on an unresolved target. Declare the edge explicitly. + # The rest of the chain (evaluate -> validate -> fixResourcePermissions -> + # execute) hangs off this action, so ordering the target here orders it for all + # of them. The reader assignment is likewise explicit because its dependency + # flows through principal_id, which Terraform cannot infer as a resource-id + # reference. + depends_on = [ + azapi_resource.workspace, + azurerm_role_assignment.reader, + azurerm_network_security_group.target, + azurerm_subnet_network_security_group_association.target, + ] +} + +# The discover LRO can report success before its pinned resource snapshot is +# visible to evaluate. A fresh measurement accepted evaluate 24 seconds after +# discover completed, while a separate run remained inconsistent beyond the +# existing 5-minute retry window. This explicit buffer handles the common case; +# evaluate's narrow retry below remains the bounded recovery for longer outliers. +resource "time_sleep" "discovery_consistency" { + create_duration = var.control_plane_consistency_wait + + depends_on = [azapi_resource_action.discover] +} + +# evaluate: assess which scenarios apply to the discovered resources. Requires a +# completed and visible discovery snapshot, so it depends on the consistency wait. +# +# Known transient error: a short "InvalidEntityState" conflict right after +# discover finishes. The AzAPI provider considers discover done once its LRO +# reports success. The Chaos control plane can take a short time after that to +# make the resulting resource snapshot visible to evaluate's own precondition +# check. In that short gap, evaluate can get a 409 with the nested detail code +# "InvalidEntityState" and a message like "no resource discovery operation has +# completed successfully". This is not a real ordering bug - Terraform's +# depends_on already waited for discover's LRO to finish. It is a short data- +# visibility gap on the control-plane side, the same kind of gap documented +# for fixResourcePermissions below, just on a different pair of actions. The +# retry block below retries only this exact nested error code. +resource "azapi_resource_action" "evaluate" { + type = local.chaos.workspace + resource_id = azapi_resource.workspace.id + action = "evaluate" + + retry = { + error_message_regex = ["(?i)\"code\":\\s*\"InvalidEntityState\""] + interval_seconds = 10 + max_interval_seconds = 60 + } + + timeouts { + create = "5m" + } + + depends_on = [time_sleep.discovery_consistency] +} + +# Evaluation has the same asynchronous visibility boundary: its LRO can finish +# before validate can read the resulting evaluation. Keep this wait separate so +# the example shows the exact action boundary even though both waits share one +# configurable duration. +resource "time_sleep" "evaluation_consistency" { + create_duration = var.control_plane_consistency_wait + + depends_on = [azapi_resource_action.evaluate] +} + +# The ScenarioConfiguration: which built-in scenario to run, its parameter +# values, and which discovered resources to include/exclude. It lives under the +# scenario it configures (parent_id = local.scenario_id). +# +# Every scenario-specific value here comes from a variable, so pointing the +# example at a different built-in scenario is a real substitution: set +# var.scenario_name, var.target_resource_types, and var.scenario_parameters +# together to match the template you name (see readme.md). +resource "azapi_resource" "scenario_configuration" { + type = local.chaos.scenario_configuration + name = "default" + parent_id = local.scenario_id + + body = { + properties = { + scenarioId = local.scenario_id + + # Runtime parameter values (KeyValuePair). Keys must match the parameters + # the chosen scenario declares; the map is projected into the wire shape. + parameters = [ + for key, value in var.scenario_parameters : { + key = key + value = value + } + ] + + # Symmetric include/exclude targeting. include pins the run to the target + # resources and the ARM types the chosen scenario can bind; exclude is + # present (empty by default) so the example teaches both sides of the + # criteria and can be narrowed without editing this file. + resourceTargeting = { + include = { + resources = local.target_resource_ids + types = var.target_resource_types + } + exclude = { + resources = var.excluded_resource_ids + } + } + } + } + + depends_on = [time_sleep.evaluation_consistency] +} + +# validate: pre-flight the configuration (resolves resources, builds the +# execution plan, surfaces permission gaps) without injecting a fault. +# +# Same structural risk as evaluate above: validate depends on evaluate having +# finished, and could see the same short "InvalidEntityState" visibility gap +# if the Chaos control plane has not yet made evaluate's result visible. This +# has not been observed for validate specifically; the retry below is added +# defensively, for the same reason, not because a failure was seen here. +resource "azapi_resource_action" "validate" { + type = local.chaos.scenario_configuration + resource_id = azapi_resource.scenario_configuration.id + action = "validate" + + retry = { + error_message_regex = ["(?i)\"code\":\\s*\"InvalidEntityState\""] + interval_seconds = 10 + max_interval_seconds = 60 + } + + timeouts { + create = "5m" + } + + depends_on = [time_sleep.evaluation_consistency] +} + +# fixResourcePermissions: assign the scenario-specific RBAC the workspace +# identity needs. The whatIf flag comes from local.fix_permissions_what_if: a +# dry-run PREVIEW (true) on the default non-executing path, and a REAL +# remediation (false) whenever var.execute_scenario is true, because a preview +# creates no assignments and the execute would then fail on missing permissions. +# A real remediation creates role assignments that Terraform does NOT track or +# remove on destroy - see readme.md for the cleanup. +resource "azapi_resource_action" "fix_resource_permissions" { + type = local.chaos.scenario_configuration + resource_id = azapi_resource.scenario_configuration.id + action = "fixResourcePermissions" + + body = { + whatIf = local.fix_permissions_what_if + } + + # Known transient error: a short 404 right after this action starts. + # + # The AzAPI provider polls this action's own status right after it sends + # the request. The Chaos control plane can take a short time to show the + # new status record. In that short gap, the poll can get "RESPONSE 404" / + # "ERROR CODE: NotFound". This is not a permission error and not a real + # failure. It is a short data-visibility gap on the control-plane side. + # + # This retry block asks Terraform to retry only that specific error text. + # It does not retry other errors. In almost all cases, the retry clears the + # 404 within one or two tries. If it does not clear, re-run apply; see + # readme.md, section "Known transient errors", for the manual recovery + # steps. + retry = { + error_message_regex = ["(?i)ERROR CODE:\\s*NotFound", "(?i)RESPONSE 404"] + interval_seconds = 10 + max_interval_seconds = 60 + } + + # Safety net for the retry above. This 5-minute limit is NOT the same thing + # as the 10-minute RBAC propagation SLA cited near time_sleep. + # permission_propagation, below. That SLA is for Azure's role-assignment + # cache, a different system. This limit only bounds how long Terraform will + # keep retrying the short control-plane data-visibility gap described above. + # In normal cases the retry clears well inside 5 minutes. If a real, non- + # transient NotFound ever matches this text by mistake, this limit still + # makes apply fail in 5 minutes, not in the provider's ~30-minute default. + timeouts { + create = "5m" + } + + # Fail at plan time rather than at run time: an explicit dry run combined with + # an execute is the one combination that always produces a broken run. + lifecycle { + precondition { + condition = !(var.execute_scenario && local.fix_permissions_what_if) + error_message = "fix_permissions_what_if = true only previews the role assignments (none are created), so the execute action would fail on missing permissions. Set fix_permissions_what_if = false, or leave it null to derive it from execute_scenario." + } + } + + depends_on = [azapi_resource_action.validate] +} + +# permission_propagation: absorb Azure RBAC propagation lag between +# fixResourcePermissions reporting Succeeded and execute actually using the +# (possibly just-created/updated) role assignments. Observed in practice across +# the automation surfaces: fixResourcePermissions can report success before the +# assignments are effective everywhere, causing execute to intermittently fail +# authorization moments later. Duration is configurable via +# permission_propagation_wait; only meaningful when execute_scenario is true +# (fix_resource_permissions still runs, as a whatIf preview, on the default path, +# so this sleep is harmless - just an unused wait - when execute is skipped). +# +# This wait is a best-effort mitigation, not a guarantee: per Microsoft's own +# documentation, Azure Resource Manager's role-assignment cache can take up to +# 10 minutes to reflect a new/updated assignment everywhere - +# https://learn.microsoft.com/en-us/azure/role-based-access-control/troubleshooting#symptom---role-assignment-changes-are-not-being-detected. +# execute can still intermittently fail authorization even with this wait in +# place; if that happens, re-running apply (fixResourcePermissions and the +# wait are safe to repeat) is the expected recovery, not a sign the example is +# broken. +resource "time_sleep" "permission_propagation" { + count = var.execute_scenario ? 1 : 0 + + create_duration = var.permission_propagation_wait + + depends_on = [azapi_resource_action.fix_resource_permissions] +} + +# execute: run the scenario, injecting real faults into the target. Gated behind +# var.execute_scenario (default false), so the default apply - including the +# upstream unattended E2E - never injects a fault. +resource "azapi_resource_action" "execute" { + count = var.execute_scenario ? 1 : 0 + + type = local.chaos.scenario_configuration + resource_id = azapi_resource.scenario_configuration.id + action = "execute" + + # Capture the created ScenarioRun's ARM id so the cancel-on-destroy action below + # can target that specific run (there is no runs/latest singleton to cancel). + response_export_values = ["id"] + + # The AzAPI provider follows the execute action's 202 + Location LRO and polls it + # until it reaches a terminal state, so this apply BLOCKS for the full run + # duration. Built-in scenarios are duration-bounded by stop-condition timers of up + # to PT3H (see readme.md), so the default 30-minute action timeout is too short. + # Raise the create timeout to cover the longest documented fault duration plus + # polling margin; tune it down if you only run short scenarios. + timeouts { + create = "3h30m" + } + + depends_on = [time_sleep.permission_propagation[0]] +} + +# cancel_run (when = destroy): best-effort cancel of the run created by execute, on +# destroy, before the workspace is torn down. Workspace delete is NOT blocked on an +# active run, and GW deletes the workspace's managed identity that fault revert +# depends on - so a run that is somehow still active at destroy time can be orphaned +# with an incomplete revert. Gated by the same var.execute_scenario, and it targets +# the run's full ARM id (under the SCENARIO, .../scenarios/{scenario}/runs/{runName}) +# captured from execute's output, not a non-existent runs/latest singleton. +# +# IMPORTANT LIMITATION - this does NOT cover an interrupted apply. The AzAPI provider +# only writes execute's output (and hence records execute in Terraform state) AFTER +# the execute LRO completes successfully. If the apply is interrupted while the run +# is still active (Ctrl-C, CI cancellation, or the execute timeout firing), execute +# never enters state, so this cancel_run - which depends on execute's output - is +# never created in state either, and a later `terraform destroy` has nothing to fire. +# On the happy path execute has already polled the run to a terminal state, so this +# cancel is a no-op. In other words cancel_run cannot rescue an interrupted apply; it +# exists for workflow completeness (FR18) and the narrow case of a still-active run at +# destroy after a *successful* apply. For an interrupted apply you must cancel the run +# manually before destroying - see the readme's "Cancel-before-destroy" section. +resource "azapi_resource_action" "cancel_run" { + count = var.execute_scenario ? 1 : 0 + + type = local.chaos.scenario_run + resource_id = azapi_resource_action.execute[0].output.id + action = "cancel" + when = "destroy" + + depends_on = [azapi_resource_action.execute] +} + +# Read the latest validation result (FR9) so outputs can surface its state. +data "azapi_resource" "latest_validation" { + type = local.chaos.latest_validation + resource_id = "${azapi_resource.scenario_configuration.id}/validations/latest" + + # The singleton-latest read can 404 before a validation materializes; stay + # 404-safe so apply never fails and the output degrades to null instead. Export + # properties.status (the Validation contract field) so latest_validation_state is + # populated rather than null. + ignore_not_found = true + response_export_values = ["properties.status"] + + depends_on = [azapi_resource_action.validate] +} + +# Read the latest permission-fix result (FR10). +data "azapi_resource" "latest_permission_fix" { + type = local.chaos.latest_permission_fix + resource_id = "${azapi_resource.scenario_configuration.id}/fixResourcePermissions/latest" + + # Same 404-safety as the validation read. The PermissionsFix contract exposes its + # lifecycle under properties.STATE (not status, which is the Validation field), so + # export properties.state here. + ignore_not_found = true + response_export_values = ["properties.state"] + + # The GET takes a whatIf query parameter: true returns the latest what-if + # (dry-run) analysis, false/absent returns the latest REAL fix result. Pass the + # mode the action above actually ran in, so the output is populated on both the + # preview and the real-remediation path instead of 404-ing on one of them. + query_parameters = { + whatIf = [tostring(local.fix_permissions_what_if)] + } + + depends_on = [azapi_resource_action.fix_resource_permissions] +} + +# Read the exact run created by this apply. Do not infer "latest" from list order: +# ARM list operations do not guarantee newest-first ordering, so value[0] can +# describe an older run after the example has been executed more than once. +data "azapi_resource" "executed_run" { + count = var.execute_scenario ? 1 : 0 + + type = local.chaos.scenario_run + resource_id = azapi_resource_action.execute[0].output.id + + response_export_values = [ + "properties.status", + "properties.errors", + "properties.executionErrors", + ] + + depends_on = [azapi_resource_action.execute] +} \ No newline at end of file diff --git a/quickstart/201-chaos-studio-scenario-run-azapi/outputs.tf b/quickstart/201-chaos-studio-scenario-run-azapi/outputs.tf new file mode 100644 index 000000000..39695195f --- /dev/null +++ b/quickstart/201-chaos-studio-scenario-run-azapi/outputs.tf @@ -0,0 +1,39 @@ +output "workspace_id" { + description = "The fully-qualified ARM id of the Chaos Studio workspace." + value = azapi_resource.workspace.id +} + +output "scenario_configuration_id" { + description = "The ARM id of the ScenarioConfiguration that drives the drill." + value = azapi_resource.scenario_configuration.id +} + +output "workspace_identity_principal_id" { + description = "The principal id of the user-assigned identity the workspace uses for discovery." + value = azurerm_user_assigned_identity.chaos.principal_id +} + +output "latest_validation_state" { + description = "State of the most recent configuration validation, or null if not yet available." + value = try(data.azapi_resource.latest_validation.output.properties.status, null) +} + +output "latest_permission_fix_state" { + description = "State of the most recent fixResourcePermissions result, or null if not yet available." + value = try(data.azapi_resource.latest_permission_fix.output.properties.state, null) +} + +output "latest_run_state" { + description = "State of the scenario run created by this apply, or null when no run has been executed (execute_scenario = false)." + value = try(data.azapi_resource.executed_run[0].output.properties.status, null) +} + +output "latest_run_errors" { + description = "System/infrastructure errors (ScenarioRunProperties.errors) recorded against the run created by this apply, or null if none/not yet available. Surfaces the API's own error detail so a Failed latest_run_state can be diagnosed without a separate ARM query." + value = try(data.azapi_resource.executed_run[0].output.properties.errors, null) +} + +output "latest_run_execution_errors" { + description = "Fault-execution errors (ScenarioRunProperties.executionErrors) recorded against the run created by this apply, or null if none/not yet available. Surfaces the API's own error detail so a Failed latest_run_state can be diagnosed without a separate ARM query." + value = try(data.azapi_resource.executed_run[0].output.properties.executionErrors, null) +} diff --git a/quickstart/201-chaos-studio-scenario-run-azapi/providers.tf b/quickstart/201-chaos-studio-scenario-run-azapi/providers.tf new file mode 100644 index 000000000..17c8b35f3 --- /dev/null +++ b/quickstart/201-chaos-studio-scenario-run-azapi/providers.tf @@ -0,0 +1,39 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + azapi = { + source = "Azure/azapi" + # >= 2.11.0: earlier azapi versions set azapi_resource_action output + # incorrectly while an action was still executing + # (Azure/terraform-provider-azapi GH-1168, fixed in 2.11.0). This example + # depends on that behavior for the discover/evaluate/validate/fix/execute + # action chain, so the floor is load-bearing here. + version = ">= 2.11.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + time = { + # Backs the control-plane consistency waits between discover/evaluate/ + # validate and the RBAC-propagation wait before execute. + source = "hashicorp/time" + version = "~> 0.9" + } + } +} + +provider "azapi" {} + +provider "azurerm" { + # azurerm v4 requires an explicit subscription id. When var.subscription_id is + # null the provider falls back to the ARM_SUBSCRIPTION_ID environment variable, + # so `az account set` / CI-provided credentials work without editing this file. + subscription_id = var.subscription_id + features {} +} diff --git a/quickstart/201-chaos-studio-scenario-run-azapi/readme.md b/quickstart/201-chaos-studio-scenario-run-azapi/readme.md new file mode 100644 index 000000000..c1bd0fd9b --- /dev/null +++ b/quickstart/201-chaos-studio-scenario-run-azapi/readme.md @@ -0,0 +1,369 @@ +# Quickstart 201 - Chaos Studio v2 scenario run (AzAPI, end-to-end) + +Provision a realistic, end-to-end Azure Chaos Studio v2 drill with Terraform and +the [AzAPI provider](https://learn.microsoft.com/en-us/azure/developer/terraform/overview-azapi-provider): +a target network security group, a user-assigned managed identity, a `Reader` +role assignment, a `Microsoft.Chaos/workspaces` resource, a ScenarioConfiguration +for a built-in scenario, and the full +`discover -> evaluate -> validate -> fixResourcePermissions -> execute` action +chain sequenced with `depends_on`. + +By default this example does **not** inject any fault: the `execute` action is +gated behind `var.execute_scenario` (default `false`), so the default +`terraform apply` provisions the target, discovers and evaluates resources, +validates the configuration, and *previews* the permission fix - then stops. +That makes the default flow safe to apply and destroy repeatedly. Set +`execute_scenario = true` to run the actual fault; on that path the permission +fix becomes a **real** remediation (see +[The action chain](#the-action-chain)), because a preview creates no role +assignments and the run would otherwise fail on missing permissions. + +For the minimal getting-started example (just a workspace, no target or run), +see the companion quickstart +[`101-chaos-studio-workspace-azapi`](../101-chaos-studio-workspace-azapi). + +## Why AzAPI instead of the azurerm provider? + +`Microsoft.Chaos` v2 is a **preview** api-version (`2026-08-01-preview`). The +`azurerm` provider has no typed resource for it, so the Chaos workspace, +configuration, and actions are expressed as raw ARM through the generic AzAPI +provider, which can target any resource type and api-version. Native `azurerm` +support is additive and does not replace these AzAPI examples - AzAPI remains how +customers reach preview api-versions. + +> **Preview caveat.** Because this targets a preview api-version, the resource +> shape and behavior can change between preview releases, and the feature must be +> enabled for your subscription and available in your region. The api-version is +> pinned in the codegen-owned `local.chaos` block in `main.tf`; when the pin +> moves, regenerate the example by running the `chaos-automation-codegen` skill +> with `surfaces: tf`. + +## Choosing a target + +Chaos Studio v2 built-in scenario templates come in two flavours, and the +difference decides what the target has to be: + +| Flavour | How the fault runs | Target prerequisites | +|---------|--------------------|----------------------| +| **Service-direct** | The fault acts on the Azure resource through its own control plane. | None beyond the resource itself. | +| **Agent-based** | Chaos Studio installs the Chaos Agent VM extension on the target's instances; the agent authenticates as the **target's own** managed identity. | A Windows/Linux VM target that carries a system- or user-assigned managed identity. | + +This quickstart deliberately defaults to a **service-direct** scenario, +`DnsOutage-1.0`, which blocks Azure DNS by temporarily adding deny rules to a +network security group. That keeps the example cheap (an NSG costs nothing), fast +(no VM to boot, no extension to install), and free of the agent's identity +prerequisite. + +> **Do not pair an agent-based scenario with a VM scale set.** Agent faults +> against `Microsoft.Compute/virtualMachineScaleSets` are **disabled in +> production**: the action provider host reads +> `ActionProviders:EnableVmssAgentFaults`, which defaults to `false` and is +> switched on only in test/canary environments. With it off, scale sets are +> filtered out of the published agent-action metadata and rejected up front by +> the validate/start target-type guards, so such a run can never execute in a +> production subscription. Agent-based scenarios in production target standalone +> `Microsoft.Compute/virtualMachines` that carry a managed identity. + +### Substituting another scenario + +`var.scenario_name` is a real substitution point: nothing about the targeting or +the parameters is hard-coded for `DnsOutage-1.0`. Change **three** variables +together so the configuration stays coherent with the template you name: + +| Variable | Must match | +|----------|-----------| +| `scenario_name` | The built-in scenario id, e.g. `EntraOutage-1.0`. | +| `target_resource_types` | The ARM types that scenario declares in its resource selectors. A type the scenario cannot bind resolves nothing and validation fails on an empty target set. | +| `scenario_parameters` | The parameters that scenario declares. Use `{}` for a scenario with none. | + +You will usually also replace the target resource in `main.tf` (or point +`var.target_resource_ids` at resources you already own - they must still sit +inside the workspace's scope). Leave `target_resource_ids` empty to target the +network security group this example creates. + +## Prerequisites + +- Terraform `>= 1.5`. +- An Azure subscription with the Chaos Studio v2 preview enabled, and a region + where it is available (set `location`). Set `ARM_SUBSCRIPTION_ID` (what + `az account set` exports) or pass `-var 'subscription_id='`. +- **Permission to create role assignments on the target scope.** This example + assigns `Reader` to the workspace identity, which requires the **deploying + principal** to hold a role with `Microsoft.Authorization/roleAssignments/write` + on the scope - for example **Owner** or **User Access Administrator** on the + subscription or resource group. A principal with only Contributor cannot create + the role assignment and the apply will fail. + +## Usage + +```bash +export ARM_SUBSCRIPTION_ID="$(az account show --query id -o tsv)" +terraform init +terraform apply +``` + +The default apply provisions and validates without injecting a fault. To run the +actual scenario: + +```bash +terraform apply -var 'execute_scenario=true' +``` + +> **Warning: running the scenario injects real faults.** With `execute_scenario = true` +> the `execute` action starts the built-in scenario (default: DNS Outage) +> against the target network security group, which adds deny rules that block +> Azure DNS and outbound UDP/53 for everything on the associated subnet for the +> configured duration. Only run this against a target you are willing to +> disrupt. Always `terraform destroy` afterward. If an apply is **interrupted** +> while a run is active, cancel the run manually before destroying (see +> "Cancel-before-destroy" below) - the `when = "destroy"` cancel action cannot +> cover an interrupted apply. +> +> **It also creates real role assignments.** On that path +> `fixResourcePermissions` runs as a real remediation (`whatIf = false`, derived +> from `execute_scenario`), so the service assigns the workspace identity the +> scenario-specific roles it needs. Terraform does not track those assignments +> and will not remove them on destroy - clean them up out of band (see the +> lifecycle caveats below). +> +> **The apply blocks until the run finishes.** The AzAPI provider follows the +> `execute` action's long-running operation and polls it to a terminal state, so +> `terraform apply` does not return until the scenario run completes. Built-in +> scenarios are duration-bounded by stop-condition timers of up to **PT3H (3 +> hours)**, so the `execute` action sets a `create` timeout of `3h30m` (the default +> action timeout of 30 minutes would abort a long run mid-flight). Lower the +> `timeouts.create` value in `main.tf` if you only run short scenarios. + +Outputs: + +| Output | Description | +|--------|-------------| +| `workspace_id` | ARM id of the Chaos Studio workspace. | +| `scenario_configuration_id` | ARM id of the ScenarioConfiguration. | +| `workspace_identity_principal_id` | Principal id of the workspace identity. | +| `latest_validation_state` | State of the most recent configuration validation. | +| `latest_permission_fix_state` | State of the most recent `fixResourcePermissions` result. The read passes the same `whatIf` mode the action ran in, so it reports the preview on the default path and the real fix on the executing path. | +| `latest_run_state` | State of the run created by this apply (null until one executes). The example reads the exact run id returned by `execute`; it does not assume ARM list order. | +| `latest_run_errors` | System/infrastructure errors (`ScenarioRunProperties.errors`) recorded against the run created by this apply, or `null` if none - use this to diagnose a `Failed` `latest_run_state` without a separate ARM query. | +| `latest_run_execution_errors` | Fault-execution errors (`ScenarioRunProperties.executionErrors`) recorded against the run created by this apply, or `null` if none. | + +## Inputs worth knowing + +| Variable | Default | Effect | +|----------|---------|--------| +| `execute_scenario` | `false` | `true` runs the fault (and makes the permission fix real). | +| `fix_permissions_what_if` | `null` (derived) | `null` = preview when not executing, real remediation when executing. `false` = create the real assignments without executing. `true` while `execute_scenario = true` is rejected at plan time. | +| `control_plane_consistency_wait` | `"60s"` | Illustrative buffer after `discover` and `evaluate` before the next action starts. A fresh measurement accepted `evaluate` 24 seconds after `discover` completed, while another run remained inconsistent beyond a five-minute retry window. Keep the wait at the action boundaries shown in `main.tf`; tune it for your environment rather than treating 60 seconds as a service guarantee. Narrow retries provide additional bounded recovery. | +| `permission_propagation_wait` | `"300s"` | How long to wait after `fixResourcePermissions` before `execute` runs, to absorb Azure RBAC propagation lag. A 120s wait was observed to be insufficient for a freshly-created managed identity even though the role assignment was already correctly in place; 300s succeeded reliably in practice. This is a best-effort mitigation, not a guarantee - Microsoft documents that Azure Resource Manager's role-assignment cache can take up to 10 minutes to be effective everywhere (see caveat below). Increase it further if `execute` still intermittently fails authorization right after a real permission fix. | +| `scenario_name` | `DnsOutage-1.0` | Which built-in scenario to configure and run. | +| `target_resource_types` | `["Microsoft.Network/networkSecurityGroups"]` | The ARM types the run may bind - must match the chosen scenario. | +| `scenario_parameters` | `{ duration = "PT15M" }` | The chosen scenario's parameter values. | +| `target_resource_ids` | `[]` | Empty = target the NSG this example creates. | +| `excluded_resource_ids` | `[]` | Resources to exclude from the run. | + +## The action chain + +The five actions are sequenced with `depends_on` to encode the required order: + +1. **discover** (on the workspace) - enumerate in-scope resources. Depends on the + workspace, the `Reader` role assignment, and + `azurerm_network_security_group.target` (plus its subnet association). The + target dependency is explicit and load-bearing: discovery + snapshots whatever is in scope *at the moment it runs*, and nothing in the + action's arguments references the NSG, so without it Terraform could create + the target after (or concurrently with) the discovery - which would then + return a scope with no target, and `validate` / `execute` would fail on an + unresolved target. Because the rest of the chain hangs off `discover`, + ordering the target here orders it for every later action. +2. **evaluate** (on the workspace) - assess which scenarios apply. Requires a + completed discovery whose snapshot is visible to the next action, so + `time_sleep.discovery_consistency` sits between `discover` and `evaluate`. + Its duration comes from `control_plane_consistency_wait` (default `60s`). + The action also carries a narrow `retry` block and a + `timeouts { create = "5m" }` limit for longer outliers - see "Known + transient errors" below. +3. **validate** (on the configuration) - pre-flight the configuration and build + the execution plan without injecting a fault. + `time_sleep.evaluation_consistency` provides the same configurable buffer + after `evaluate`. `validate` carries the same kind of narrow retry block, + added defensively for the same reason - see "Known transient errors" below. +4. **fixResourcePermissions** (on the configuration) - assign the + scenario-specific RBAC the workspace identity needs. Its `whatIf` flag is + derived from `execute_scenario` unless `fix_permissions_what_if` is set + explicitly: a dry-run **preview** (`whatIf = true`, no assignments created) on + the default non-executing path, and a **real remediation** + (`whatIf = false`) when `execute_scenario = true` - a preview creates nothing, + so an execute after one would fail on missing permissions. A + `lifecycle.precondition` rejects `fix_permissions_what_if = true` combined + with `execute_scenario = true` at plan time. This action carries its own + `retry` block and a `timeouts { create = "5m" }` limit - see "Known + transient errors" below. +5. **execute** (on the configuration) - run the scenario. Gated by + `var.execute_scenario`. The provider polls this action's LRO to a terminal + state, so `apply` blocks for the run's full duration (see the timeout note in + the warning above). A `time_sleep.permission_propagation` resource + (`permission_propagation_wait`, default `300s`) is created only on the + executing path and sits between + `fixResourcePermissions` and `execute`: `fixResourcePermissions` reporting + `Succeeded` does not guarantee the role assignments are effective everywhere + yet, and without a wait `execute` can intermittently fail authorization + moments after a real permission fix. A shorter (120s) wait was observed to + be insufficient in practice even with the role assignment already correctly + in place; increase this further if authorization failures persist. + + **This wait is a best-effort mitigation, not a guarantee.** Per + [Microsoft's documentation on Azure RBAC troubleshooting](https://learn.microsoft.com/en-us/azure/role-based-access-control/troubleshooting#symptom---role-assignment-changes-are-not-being-detected), + Azure Resource Manager's role-assignment cache can take **up to 10 minutes** + to reflect a new or updated assignment everywhere. `execute` can therefore + still intermittently fail authorization even with this wait in place. If + that happens, re-running `apply` is the expected recovery (both + `fixResourcePermissions` and the wait are safe to repeat) - it isn't a sign + the example itself is broken. + +## Known transient errors + +- **A short `409 Conflict` / `InvalidEntityState` on `evaluate` (and, + defensively, `validate`).** Right after `discover` (or `evaluate`) finishes, + the Chaos control plane can take a short time to make that action's result + visible to the next action's own precondition check. In that short gap, + `evaluate` can get a 409 with the nested detail code `InvalidEntityState` + and a message like "no resource discovery operation has completed + successfully". This is not a real ordering bug - Terraform's `depends_on` + already waited for the prior action's LRO to finish. It is a short + data-visibility gap on the control-plane side. The explicit + `time_sleep.discovery_consistency` and `time_sleep.evaluation_consistency` + resources make both boundaries visible and configurable in the example. + Their shared default is 60 seconds: enough for the measured 24-second + success case, but not a service guarantee. `evaluate` and `validate` also + carry a `retry` block that retries only this exact nested error code, and a + `timeouts { create = "5m" }` limit as a safety net for longer outliers. + Production automation should tune these waits from its own telemetry rather + than copy the quickstart default without measurement. +- **A short `404 Not Found` on `fixResourcePermissions`.** Right after this + action starts, the AzAPI provider polls the action's own status. The Chaos + control plane can take a short time to show the new status record. In that + short gap, the poll can get `RESPONSE 404` / `ERROR CODE: NotFound`. This is + not a permission error. It is a short data-visibility gap on the + control-plane side, for the `fixResourcePermissions` status record only. It + is **not** the same thing as the RBAC role-assignment propagation covered + above; that is a different Azure system (the `Microsoft.Authorization` role + cache) with a different, much longer SLA. This example's + `fix_resource_permissions` action carries a `retry` block that retries only + this exact error text, and a `timeouts { create = "5m" }` limit as a safety + net. In almost all cases the retry clears the 404 within the first one or + two tries. If `apply` still fails with this error after 5 minutes, re-run + `apply` - this action is safe to repeat. + +## AzAPI action lifecycle caveats + +- **Actions fire on create/replace only.** An `azapi_resource_action` runs when + it is first created (or replaced); it does **not** re-fire on a subsequent + unchanged `terraform apply`. To re-run an action, taint or replace it + (`terraform apply -replace=
`). In particular, re-running the scenario + requires replacing the `execute` action, not just re-applying. (Changing an + action's `body` *is* a change, so flipping `execute_scenario` to `true` re-runs + `fixResourcePermissions` - its `whatIf` body flips with it - before `execute` + is created.) +- **`terraform destroy` performs no action** unless the action block sets + `when = "destroy"` (only the `cancel_run` block does). +- **`fixResourcePermissions` with `whatIf = false` creates real role + assignments** that Terraform does not track in state and will not remove on + destroy. That is the mode the executing path uses (`execute_scenario = true` + derives `whatIf = false`), and you can opt into it without executing by setting + `fix_permissions_what_if = false`. Clean up the assignments it created out of + band - they survive `terraform destroy` of everything else, though deleting the + workspace identity leaves them orphaned rather than usable. +- **The target NSG declares no `security_rule` blocks.** The deny rules the fault + adds and reverts are service-managed; leaving the attribute unset keeps + Terraform from trying to reconcile them if you happen to run a plan while a run + is active. + +## Why cancel-before-destroy + +`main.tf` includes a `when = "destroy"` `azapi_resource_action` that cancels the +scenario run, gated by the same `var.execute_scenario`. It exists because +**the service does not block workspace deletion on an active run**: + +- `Workspace.EnsureDeletionAllowed()` guards only active discovery/evaluation + operations, not runs, and neither BE nor GW cascade-cancels ScenarioRuns on + workspace delete. +- GW's `DeleteOrchestration` also deletes the workspace's managed identities - + the very identities a fault's revert step depends on. + +So a `terraform destroy` while a run is active returns success while leaving an +orphaned run whose fault revert may fail once the control-plane resources are +gone. Fault steps are duration-bounded (stop-condition timers, PT1M-PT3H) and +continuous actions have a stop path, but revert is not guaranteed after the +control plane is removed - hence a `when = "destroy"` cancel action. + +> **Limitation - the cancel action cannot rescue an interrupted apply.** The +> AzAPI provider only writes the `execute` action's output to Terraform state +> **after** the execute LRO completes successfully. So on the happy path (execute +> polled the run to a terminal state) there is no active run and the cancel is a +> no-op; and on an **interrupted** apply - `Ctrl-C`, CI cancellation, or the +> `execute` timeout firing while the run is still active - `execute` never enters +> state, the dependent `cancel_run` is never created in state, and a later +> `terraform destroy` has nothing to fire. The cancel action exists for workflow +> completeness (FR18) and the narrow case of a still-active run at destroy after a +> successful apply; it is **not** a guarantee against interruption. +> +> **Manual remediation for an interrupted apply.** If an apply is interrupted +> while a scenario run is active, cancel the run yourself and wait for it to reach +> a terminal state **before** destroying, so the fault reverts while the workspace +> identity still exists. Run this Bash from the example directory: +> +> ```bash +> # Derive the scenario scope from the Terraform output. The configuration id is +> # .../scenarios/{scenario}/configurations/{config}; runs live under the SCENARIO. +> SCENARIO_CONFIGURATION_ID=$(terraform output -raw scenario_configuration_id) +> SCENARIO_ID="${SCENARIO_CONFIGURATION_ID%/configurations/*}" +> +> # 1. List the runs under the scenario and pick the active one. +> az rest --method get \ +> --url "${SCENARIO_ID}/runs?api-version=2026-08-01-preview" \ +> --query "value[].{id:id, status:properties.status}" -o table +> +> # 2. Request cancellation of that run (RUN_ID is its full ARM id from the list). +> RUN_ID="" +> az rest --method post \ +> --url "${RUN_ID}/cancel?api-version=2026-08-01-preview" +> +> # 3. Cancel is asynchronous. Poll until the run is TERMINAL (Succeeded, Failed, +> # or Canceled - note "Canceling" is NOT terminal) before you destroy. +> while true; do +> status=$(az rest --method get \ +> --url "${RUN_ID}?api-version=2026-08-01-preview" \ +> --query "properties.status" -o tsv) +> echo "run status: ${status}" +> case "${status}" in +> Succeeded|Failed|Canceled) break ;; +> esac +> sleep 10 +> done +> ``` +> +> Only once the run is terminal is it safe to run `terraform destroy`. (You can +> also cancel the run from the Chaos Studio portal, but still wait for a terminal +> state before destroying.) + +## What this creates + +| Resource | Purpose | +|----------|---------| +| `random_pet.this` | Readable name suffix. | +| `azurerm_resource_group.this` | Holds every resource; the workspace's scope. | +| `azurerm_virtual_network.this`, `azurerm_subnet.this` | Network the target NSG protects. | +| `azurerm_network_security_group.target` | The fault target. Service-direct, so it needs no agent and no managed identity of its own. | +| `azurerm_subnet_network_security_group_association.target` | Attaches the NSG to the subnet so the fault's deny rules have a real blast radius. | +| `azurerm_user_assigned_identity.chaos` | Identity the workspace uses for discovery, permission fixing, and fault execution. | +| `azurerm_role_assignment.reader` | Grants that identity `Reader` on the scope. | +| `azapi_resource.workspace` | The `Microsoft.Chaos/workspaces` resource. | +| `azapi_resource.scenario_configuration` | The ScenarioConfiguration to run. | +| `azapi_resource_action.{discover,evaluate,validate,fix_resource_permissions,execute,cancel_run}` | The action chain. | +| `data.azapi_resource.{latest_validation,latest_permission_fix}`, `data.azapi_resource_list.runs` | Result reads for outputs. | + +The `Reader` role definition GUID (`acdd72a7-3385-48ef-bd42-f606fba81ae7`) is +verified against the +[Azure built-in role catalog](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/general#reader). +Scenario-specific RBAC is delegated to `fixResourcePermissions` rather than +hand-coded broad grants; no `Owner`/`Contributor` shortcut is used. \ No newline at end of file diff --git a/quickstart/201-chaos-studio-scenario-run-azapi/variables.tf b/quickstart/201-chaos-studio-scenario-run-azapi/variables.tf new file mode 100644 index 000000000..988bd0ac7 --- /dev/null +++ b/quickstart/201-chaos-studio-scenario-run-azapi/variables.tf @@ -0,0 +1,84 @@ +variable "subscription_id" { + description = "Azure subscription ID to deploy into. The azurerm v4 provider requires a subscription id; when this is left null it is sourced from the ARM_SUBSCRIPTION_ID environment variable (which `az account set` and most CI set automatically). Set it explicitly to target a specific subscription. (The Reader role-definition id in main.tf is derived separately from the data.azurerm_subscription.current data source, not from this variable.)" + type = string + default = null +} + +variable "location" { + description = "Azure region for the resource group, target, and workspace. Must be a region where the Microsoft.Chaos v2 preview is enabled." + type = string + default = "eastus" +} + +variable "name_prefix" { + description = "Prefix for generated resource names. A random pet suffix is appended for uniqueness." + type = string + default = "chaos-drill" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,20}$", var.name_prefix)) + error_message = "name_prefix must be 2-21 characters: lowercase letters, digits, or hyphens, starting with a letter." + } +} + +variable "tags" { + description = "Tags applied to every resource this quickstart creates." + type = map(string) + default = {} +} + +variable "scenario_name" { + description = "The built-in Chaos Studio v2 scenario to run against the target. The default (DnsOutage-1.0) is a SERVICE-DIRECT template that blocks Azure DNS by temporarily adding deny rules to a network security group - no Chaos Agent, and no managed identity on the target. The scenario is referenced as a child of the workspace (workspaces/{ws}/scenarios/{scenario_name}). To substitute another built-in scenario, change this together with target_resource_types (the ARM types that scenario declares) and scenario_parameters (the parameters it declares) - see readme.md." + type = string + default = "DnsOutage-1.0" +} + +variable "scenario_parameters" { + description = "Runtime parameter values for the chosen scenario, as a key/value map projected into the ScenarioConfiguration's KeyValuePair list. Keys MUST match parameters the scenario declares; the default supplies the DnsOutage-1.0 'duration' parameter. Set {} for a scenario that declares no parameters." + type = map(string) + default = { + duration = "PT15M" + } +} + +variable "target_resource_types" { + description = "ARM resource types the run may bind, written to resourceTargeting.include.types. MUST be types the chosen scenario declares in its ResourceSelectors, or the configuration resolves nothing and validation fails on an empty target set. The default matches DnsOutage-1.0." + type = list(string) + default = ["Microsoft.Network/networkSecurityGroups"] +} + +variable "target_resource_ids" { + description = "Explicit ARM ids to target, written to resourceTargeting.include.resources. Leave empty (the default) to target the network security group this quickstart creates; set it to point the same scenario at resources you already own (they must still be inside the workspace's scope)." + type = list(string) + default = [] +} + +variable "excluded_resource_ids" { + description = "ARM ids to exclude from the run, written to resourceTargeting.exclude.resources. Empty by default - the key is always emitted so the example teaches both sides of the include/exclude criteria." + type = list(string) + default = [] +} + +variable "execute_scenario" { + description = "When false (the default) the example provisions the workspace, discovers/evaluates, validates the configuration, and PREVIEWS the permission fix WITHOUT injecting any fault - safe to apply and destroy repeatedly. Set to true to actually execute the scenario, which injects real faults into the target (the default scenario temporarily applies DNS-blocking deny rules to the target network security group) and (via the derived fix_permissions_what_if) creates the real role assignments the run needs. A best-effort cancel-on-destroy action (see main.tf) cancels a run that is still active at destroy time after a successful apply; it does NOT cover an interrupted apply - in that case cancel the run manually and wait for a terminal state before destroying (see readme.md)." + type = bool + default = false +} + +variable "control_plane_consistency_wait" { + description = "How long to wait (a time_sleep duration string, e.g. \"60s\") after discover and evaluate report success before starting the next action. These Chaos control-plane actions can complete their LRO before the result is visible to the next action's precondition check. The default is an illustrative buffer for this quickstart, not a service guarantee; increase it if your environment still returns transient InvalidEntityState conflicts. Narrow retries in main.tf provide additional bounded recovery." + type = string + default = "60s" +} + +variable "permission_propagation_wait" { + description = "How long to wait (a time_sleep duration string, e.g. \"300s\") after fixResourcePermissions before execute runs, to absorb Azure RBAC propagation lag. fixResourcePermissions reporting Succeeded does not guarantee the new/updated role assignments are already effective everywhere; without a wait, execute can intermittently fail authorization even though permissions were 'fixed' moments earlier. Observed in practice: a 120s wait was NOT sufficient for a freshly-created managed identity (execute still failed authorization on the exact resource fixResourcePermissions had just granted access to); 300s is the current default. Only relevant when execute_scenario is true. Increase it further if execute still hits transient authorization failures; the AzAPI execute action's own 3h30m timeout has ample headroom for a longer wait." + type = string + default = "300s" +} + +variable "fix_permissions_what_if" { + description = "Controls the whatIf flag on the fixResourcePermissions action. Leave null (the default) to derive it from execute_scenario: a dry-run preview (whatIf = true) when execute_scenario is false, and a REAL remediation (whatIf = false) when execute_scenario is true - a preview creates no role assignments, so an execute after one would fail on missing permissions. Set false explicitly to create the real assignments without executing. Setting true while execute_scenario is true is rejected at plan time by a precondition in main.tf. A real remediation creates role assignments Terraform does not track or remove on destroy (see readme.md)." + type = bool + default = null +} \ No newline at end of file