State Manipulation

8 minute read

State Manipulation Overview

Beyond basic CRUD operations, Terraform provides tools for advanced state management. Your provider must handle these correctly.

Advanced state operations:

Remote state backends
State locking
State versioning
Partial state updates
Manual state commands
State refactoring

Private State Data

What is Private Data?

Provider-specific metadata not exposed to users but stored in state:

{
  "instances": [
    {
      "schema_version": 1,
      "attributes": { /* ... */ },
      "private": "eyJpbnRlcm5hbF9pZCI6IjEyMzQ1In0="  // ← Base64 encoded
    }
  ]
}

Storing Private Data

func (r *DatabaseResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) {
    var plan DatabaseResourceModel
    req.Plan.Get(ctx, &plan)
    
    // Create database
    db, err := r.client.CreateDatabase(ctx, input)
    if err != nil {
        resp.Diagnostics.AddError("Create Failed", err.Error())
        return
    }
    
    // Set public state
    state := DatabaseResourceModel{
        ID:   types.StringValue(db.ID),
        Name: types.StringValue(db.Name),
    }
    resp.State.Set(ctx, state)
    
    // Store private metadata
    privateData := map[string]interface{}{
        "internal_id":       db.InternalID,        // Internal tracking ID
        "provisioning_token": db.ProvisionToken,    // Temporary token
        "api_version":       "v2",                 // API version used
    }
    
    privateJSON, _ := json.Marshal(privateData)
    resp.Private.SetKey(ctx, "database_metadata", privateJSON)
    
    tflog.Debug(ctx, "Stored private metadata", map[string]any{
        "internal_id": db.InternalID,
    })
}

Retrieving Private Data

func (r *DatabaseResource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) {
    var plan, state DatabaseResourceModel
    req.Plan.Get(ctx, &plan)
    req.State.Get(ctx, &state)
    
    // Retrieve private data
    privateJSON, diags := req.Private.GetKey(ctx, "database_metadata")
    resp.Diagnostics.Append(diags...)
    
    var privateData map[string]interface{}
    if len(privateJSON) > 0 {
        json.Unmarshal(privateJSON, &privateData)
        
        tflog.Debug(ctx, "Retrieved private metadata", map[string]any{
            "internal_id": privateData["internal_id"],
        })
        
        // Use internal ID for API call if needed
        if internalID, ok := privateData["internal_id"].(string); ok {
            // Make more efficient API call using internal ID
            db, err := r.client.UpdateDatabaseByInternalID(ctx, internalID, input)
            // ...
        }
    }
    
    // Update resource...
}

Use Cases for Private Data

1. Internal IDs:

// Store internal ID for efficient lookups
privateData := map[string]interface{}{
    "internal_id": db.InternalID,  // Not exposed to users
}

2. Provisioning State:

// Track multi-step provisioning
privateData := map[string]interface{}{
    "provisioning_step": 2,
    "provisioning_token": token,
}

3. API Version Tracking:

// Track which API version created resource
privateData := map[string]interface{}{
    "api_version": "v2",
    "created_by":  "terraform-provider-mycloud/2.0.0",
}

4. Caching Computed Values:

// Cache expensive computations
privateData := map[string]interface{}{
    "cached_topology": topologyData,
    "cache_timestamp": time.Now().Unix(),
}

State Import Advanced Patterns

Import with Validation

func (r *DatabaseResource) ImportState(ctx context.Context, req resource.ImportStateRequest, resp *resource.ImportStateResponse) {
    id := req.ID
    
    // Validate ID format
    if !regexp.MustCompile(`^db-[a-z0-9]{8}$`).MatchString(id) {
        resp.Diagnostics.AddError(
            "Invalid Database ID Format",
            fmt.Sprintf(
                "Database ID must match pattern 'db-XXXXXXXX', got: %s\n\n"+
                "Find valid IDs with: mycloud-cli list-databases",
                id,
            ),
        )
        return
    }
    
    // Fetch and validate existence
    db, err := r.client.GetDatabase(ctx, id)
    if err != nil {
        if client.IsNotFoundError(err) {
            resp.Diagnostics.AddError(
                "Database Not Found",
                fmt.Sprintf("No database exists with ID: %s", id),
            )
            return
        }
        resp.Diagnostics.AddError("Import Failed", err.Error())
        return
    }
    
    // Check if database is in importable state
    if db.Status == "deleting" || db.Status == "failed" {
        resp.Diagnostics.AddError(
            "Cannot Import Database",
            fmt.Sprintf(
                "Database %s is in '%s' state and cannot be imported.\n"+
                "Only databases in 'available' or 'stopped' state can be imported.",
                id, db.Status,
            ),
        )
        return
    }
    
    // Set ID for Read to populate
    resource.ImportStatePassthroughID(ctx, path.Root("id"), req, resp)
    
    tflog.Info(ctx, "Validated and imported database", map[string]any{
        "id":     db.ID,
        "status": db.Status,
    })
}

Import with Private Data

func (r *DatabaseResource) ImportState(ctx context.Context, req resource.ImportStateRequest, resp *resource.ImportStateResponse) {
    id := req.ID
    
    // Fetch database
    db, err := r.client.GetDatabase(ctx, id)
    if err != nil {
        resp.Diagnostics.AddError("Import Failed", err.Error())
        return
    }
    
    // Set ID
    resource.ImportStatePassthroughID(ctx, path.Root("id"), req, resp)
    
    // Store private metadata during import
    privateData := map[string]interface{}{
        "internal_id":  db.InternalID,
        "imported_at":  time.Now().Unix(),
        "api_version":  "v2",
    }
    
    privateJSON, _ := json.Marshal(privateData)
    resp.Private.SetKey(ctx, "database_metadata", privateJSON)
    
    tflog.Info(ctx, "Imported database with private metadata")
}

State Dependencies

Explicit Dependencies

Resources can reference each other:

resource "mycloud_vpc" "main" {
  cidr_block = "10.0.0.0/16"
}

resource "mycloud_subnet" "app" {
  vpc_id     = mycloud_vpc.main.id  # Dependency
  cidr_block = "10.0.1.0/24"
}

In state:

{
  "instances": [
    {
      "attributes": {
        "id": "subnet-123",
        "vpc_id": "vpc-456"
      },
      "dependencies": [
        "mycloud_vpc.main"  // ← Terraform tracks this
      ]
    }
  ]
}

Your Provider’s Role

Framework handles dependencies automatically. You just need to:

// In subnet resource
"vpc_id": schema.StringAttribute{
    Required: true,
},

// Terraform will:
// 1. Create VPC first
// 2. Pass VPC ID to subnet
// 3. Track dependency in state

Partial State Updates

Updating Specific Attributes

func (r *DatabaseResource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) {
    var plan, state DatabaseResourceModel
    req.Plan.Get(ctx, &plan)
    req.State.Get(ctx, &state)
    
    // Check what changed
    if !plan.StorageGB.Equal(state.StorageGB) {
        tflog.Info(ctx, "Updating storage", map[string]any{
            "from": state.StorageGB.ValueInt64(),
            "to":   plan.StorageGB.ValueInt64(),
        })
        
        // Update via API
        err := r.client.UpdateDatabaseStorage(ctx, state.ID.ValueString(), UpdateStorageInput{
            StorageGB: int(plan.StorageGB.ValueInt64()),
        })
        
        if err != nil {
            resp.Diagnostics.AddError("Update Failed", err.Error())
            return
        }
        
        // Update specific attribute in state
        state.StorageGB = plan.StorageGB
    }
    
    // Check tags
    if !plan.Tags.Equal(state.Tags) {
        tflog.Info(ctx, "Updating tags")
        
        var planTags, stateTags map[string]string
        plan.Tags.ElementsAs(ctx, &planTags, false)
        state.Tags.ElementsAs(ctx, &stateTags, false)
        
        // Update via API
        err := r.client.UpdateDatabaseTags(ctx, state.ID.ValueString(), planTags)
        if err != nil {
            resp.Diagnostics.AddError("Tag Update Failed", err.Error())
            return
        }
        
        // Update tags in state
        state.Tags = plan.Tags
    }
    
    // Save updated state
    resp.State.Set(ctx, state)
}

State Locking (Backend Responsibility)

Provider Doesn’t Handle Locking

State locking is handled by backends, not providers:

terraform {
  backend "s3" {
    bucket         = "my-state-bucket"
    key            = "prod/terraform.tfstate"
    region         = "us-east-1"
    dynamodb_table = "terraform-locks"  # ← Locking
    encrypt        = true
  }
}

Idempotent Operations

Ensure your provider operations are idempotent:

func (r *DatabaseResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) {
    var plan DatabaseResourceModel
    req.Plan.Get(ctx, &plan)
    
    // Create database
    db, err := r.client.CreateDatabase(ctx, input)
    
    if err != nil {
        // Check if already exists (retry scenario)
        if client.IsAlreadyExistsError(err) {
            tflog.Warn(ctx, "Database already exists, fetching existing")
            
            // Fetch existing database
            existingDB, fetchErr := r.client.GetDatabaseByName(ctx, plan.Name.ValueString())
            if fetchErr != nil {
                resp.Diagnostics.AddError("Create Failed", err.Error())
                return
            }
            
            db = existingDB
        } else {
            resp.Diagnostics.AddError("Create Failed", err.Error())
            return
        }
    }
    
    // Set state
    state := DatabaseResourceModel{
        ID:   types.StringValue(db.ID),
        Name: types.StringValue(db.Name),
    }
    resp.State.Set(ctx, state)
}

Remote State

Reading Remote State (Data Source)

Users can read state from other workspaces:

# Workspace A: Creates VPC
resource "mycloud_vpc" "main" {
  cidr_block = "10.0.0.0/16"
}

# Workspace B: Reads VPC from A's state
data "terraform_remote_state" "network" {
  backend = "s3"
  config = {
    bucket = "my-state-bucket"
    key    = "network/terraform.tfstate"
    region = "us-east-1"
  }
}

resource "mycloud_instance" "app" {
  vpc_id = data.terraform_remote_state.network.outputs.vpc_id
}

Provider responsibility: Expose useful attributes as outputs in your documentation.

State Outputs

# Encourage users to output useful values
output "vpc_id" {
  value       = mycloud_vpc.main.id
  description = "VPC ID for cross-workspace reference"
}

output "vpc_cidr" {
  value = mycloud_vpc.main.cidr_block
}

Manual State Operations

Users may manually manipulate state. Your provider must handle this gracefully.

State Move

# Rename resource
$ terraform state mv mycloud_database.old mycloud_database.new

# Move to module
$ terraform state mv mycloud_database.db module.databases.mycloud_database.db

Your provider: No special handling needed. Framework handles this.

State Remove

# Remove from state (doesn't delete resource)
$ terraform state rm mycloud_database.test

Your provider: Next apply will detect resource exists and may try to recreate it.

State Import

# Import existing resource
$ terraform import mycloud_database.db db-12345

Your provider: Implement ImportState method (covered earlier).

State Backup

Automatic Backups

Terraform creates .tfstate.backup:

$ ls -la
terraform.tfstate
terraform.tfstate.backup  # ← Previous version

$ terraform apply
# State updated, backup created

Manual Backup

# Pull state
$ terraform state pull > backup-$(date +%Y%m%d).tfstate

# Restore if needed
$ terraform state push backup-20260701.tfstate

Provider responsibility: None. This is handled by Terraform CLI.

State Encryption

Backend Encryption

terraform {
  backend "s3" {
    bucket  = "my-state-bucket"
    key     = "prod/terraform.tfstate"
    region  = "us-east-1"
    encrypt = true  # ← Server-side encryption
    
    kms_key_id = "arn:aws:kms:us-east-1:123:key/abc"
  }
}

Sensitive Attributes

Mark sensitive data in schema:

"password": schema.StringAttribute{
    Required:  true,
    Sensitive: true,  // ← Hidden in logs, marked in state
},

In state:

{
  "attributes": {
    "password": "secret123"
  },
  "sensitive_attributes": [
    {
      "type": "get_attr",
      "value": "password"
    }
  ]
}

Never log sensitive values:

// ❌ Bad: Logs password
tflog.Debug(ctx, "Creating database", map[string]any{
    "password": plan.Password.ValueString(),
})

// ✅ Good: Don't log sensitive data
tflog.Debug(ctx, "Creating database", map[string]any{
    "name": plan.Name.ValueString(),
    // Don't log password!
})

State Versioning

State Serial Number

Terraform increments serial on each write:

{
  "version": 4,
  "serial": 42,  // ← Increments each apply
  "lineage": "abc-123",
  "resources": [...]
}

Lineage

UUID identifying state file lineage:

{
  "lineage": "8f3a4b2c-1d5e-4f6a-9b2c-3d4e5f6a7b8c",
  // Same lineage = same state history
}

Provider responsibility: None. Terraform handles versioning.

Best Practices

1. Use Private Data Sparingly

// ✅ Good: Only store what's needed
privateData := map[string]interface{}{
    "internal_id": db.InternalID,
}

// ❌ Bad: Storing unnecessary data
privateData := map[string]interface{}{
    "internal_id": db.InternalID,
    "full_api_response": entireAPIResponse,  // Too much!
    "cache": expensiveComputation,           // Use computed attrs
}

2. Handle State Corruption Gracefully

// ✅ Good: Validate state data
func (r *DatabaseResource) Read(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) {
    var state DatabaseResourceModel
    req.State.Get(ctx, &state)
    
    // Validate ID exists
    if state.ID.IsNull() || state.ID.ValueString() == "" {
        resp.Diagnostics.AddError(
            "Invalid State",
            "Database ID is missing from state. Resource may need to be reimported.",
        )
        return
    }
    
    // Continue normally...
}

3. Log State Operations

// ✅ Good: Log for debugging
tflog.Debug(ctx, "Storing private metadata", map[string]any{
    "keys": []string{"internal_id", "api_version"},
})

tflog.Info(ctx, "Updated state", map[string]any{
    "id":         db.ID,
    "attributes": []string{"storage_gb", "tags"},
})

4. Document State Schema

// ✅ Good: Document what's stored
resp.Schema = schema.Schema{
    MarkdownDescription: "Manages a database cluster.\n\n" +
        "## State Storage\n\n" +
        "This resource stores the following in state:\n" +
        "- All configurable attributes\n" +
        "- Computed attributes (ID, endpoint, ARN)\n" +
        "- Private metadata (internal tracking IDs)\n\n" +
        "Sensitive attributes (password) are marked as sensitive in state.",
    
    Attributes: map[string]schema.Attribute{
        // ...
    },
}

5. Test State Operations

func TestDatabaseResource_StateManipulation(t *testing.T) {
    resource.Test(t, resource.TestCase{
        ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
        Steps: []resource.TestStep{
            {
                Config: testAccDatabaseConfig("test"),
            },
            {
                // Test state move
                ResourceName: "mycloud_database.test",
                Config:       testAccDatabaseConfig("test"),
                PreConfig: func() {
                    // Move resource in state
                    // terraform state mv ...
                },
            },
        },
    })
}

Key Takeaways

✅ Use private data for provider-specific metadata
✅ Private data not exposed to users but stored in state
✅ State locking handled by backends, not providers
✅ Make operations idempotent for retry scenarios
✅ Mark sensitive attributes in schema
✅ Never log sensitive values
✅ Validate state data in Read operation
✅ Handle state corruption gracefully
✅ Document what’s stored in state
✅ Test state operations and edge cases

What’s Next?

Put everything together in a comprehensive lab!

Next: State Management Lab →

Previous: ← Schema Upgrades

Last updated: July 1, 2026

Share on

Twitter Facebook LinkedIn