Data Sources Lab

10 minute read

Lab Overview

In this hands-on lab, you’ll build data sources for the CacheCloud provider:

What You’ll Build:

  • Single result data source (mycloud_cache)
  • List data source (mycloud_caches)
  • Filtering support
  • Multiple lookup methods

Time: 60-75 minutes

Prerequisites

Continue with the CacheCloud provider from Module 6 Resource Implementation Lab.

cd terraform-provider-cachecloud

Step 1: Single Cache Data Source

Create internal/provider/cache_data_source.go:

package provider

import (
    "context"
    "fmt"
    
    "github.com/hashicorp/terraform-plugin-framework/attr"
    "github.com/hashicorp/terraform-plugin-framework/datasource"
    "github.com/hashicorp/terraform-plugin-framework/datasource/schema"
    "github.com/hashicorp/terraform-plugin-framework/types"
    "github.com/hashicorp/terraform-plugin-log/tflog"
    
    "github.com/yourorg/terraform-provider-cachecloud/internal/client"
)

type CacheDataSource struct {
    client *client.Client
}

type CacheDataSourceModel struct {
    ID            types.String `tfsdk:"id"`
    Name          types.String `tfsdk:"name"`
    Engine        types.String `tfsdk:"engine"`
    EngineVersion types.String `tfsdk:"engine_version"`
    NodeType      types.String `tfsdk:"node_type"`
    NumNodes      types.Int64  `tfsdk:"num_nodes"`
    Port          types.Int64  `tfsdk:"port"`
    Status        types.String `tfsdk:"status"`
    Endpoint      types.String `tfsdk:"endpoint"`
    ARN           types.String `tfsdk:"arn"`
    Tags          types.Map    `tfsdk:"tags"`
}

var _ datasource.DataSource = &CacheDataSource{}
var _ datasource.DataSourceWithConfigure = &CacheDataSource{}

func NewCacheDataSource() datasource.DataSource {
    return &CacheDataSource{}
}

func (d *CacheDataSource) Metadata(ctx context.Context, req datasource.MetadataRequest, resp *datasource.MetadataResponse) {
    resp.TypeName = req.ProviderTypeName + "_cache"
}

func (d *CacheDataSource) Schema(ctx context.Context, req datasource.SchemaRequest, resp *datasource.SchemaResponse) {
    resp.Schema = schema.Schema{
        MarkdownDescription: "Fetches information about a single cache cluster",
        
        Attributes: map[string]schema.Attribute{
            "id": schema.StringAttribute{
                Optional:            true,
                Computed:            true,
                MarkdownDescription: "Cache cluster ID. Conflicts with `name`",
            },
            "name": schema.StringAttribute{
                Optional:            true,
                Computed:            true,
                MarkdownDescription: "Cache cluster name. Conflicts with `id`",
            },
            "engine": schema.StringAttribute{
                Computed:            true,
                MarkdownDescription: "Cache engine (redis or memcached)",
            },
            "engine_version": schema.StringAttribute{
                Computed:            true,
                MarkdownDescription: "Engine version",
            },
            "node_type": schema.StringAttribute{
                Computed:            true,
                MarkdownDescription: "Node type",
            },
            "num_nodes": schema.Int64Attribute{
                Computed:            true,
                MarkdownDescription: "Number of cache nodes",
            },
            "port": schema.Int64Attribute{
                Computed:            true,
                MarkdownDescription: "Connection port",
            },
            "status": schema.StringAttribute{
                Computed:            true,
                MarkdownDescription: "Current status",
            },
            "endpoint": schema.StringAttribute{
                Computed:            true,
                MarkdownDescription: "Cache endpoint",
            },
            "arn": schema.StringAttribute{
                Computed:            true,
                MarkdownDescription: "ARN of the cache cluster",
            },
            "tags": schema.MapAttribute{
                Optional:            true,
                Computed:            true,
                ElementType:         types.StringType,
                MarkdownDescription: "Tags to filter by or returned tags",
            },
        },
    }
}

func (d *CacheDataSource) Configure(ctx context.Context, req datasource.ConfigureRequest, resp *datasource.ConfigureResponse) {
    if req.ProviderData == nil {
        return
    }
    
    client, ok := req.ProviderData.(*client.Client)
    if !ok {
        resp.Diagnostics.AddError(
            "Unexpected Data Source Configure Type",
            fmt.Sprintf("Expected *client.Client, got: %T", req.ProviderData),
        )
        return
    }
    
    d.client = client
}

func (d *CacheDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config CacheDataSourceModel
    diags := req.Config.Get(ctx, &config)
    resp.Diagnostics.Append(diags...)
    if resp.Diagnostics.HasError() {
        return
    }
    
    // Validate that at least one search criterion is provided
    hasID := !config.ID.IsNull()
    hasName := !config.Name.IsNull()
    hasTags := !config.Tags.IsNull()
    
    if !hasID && !hasName && !hasTags {
        resp.Diagnostics.AddError(
            "Missing Search Criteria",
            "One of 'id', 'name', or 'tags' must be specified",
        )
        return
    }
    
    if hasID && hasName {
        resp.Diagnostics.AddError(
            "Conflicting Search Criteria",
            "Only one of 'id' or 'name' can be specified",
        )
        return
    }
    
    var cache *client.Cache
    var err error
    
    // Lookup by ID
    if hasID {
        id := config.ID.ValueString()
        tflog.Debug(ctx, "Looking up cache by ID", map[string]any{"id": id})
        
        cache, err = d.client.GetCache(ctx, id)
        if err != nil {
            resp.Diagnostics.AddError(
                "Cache Not Found",
                fmt.Sprintf("Could not find cache with ID '%s': %s", id, err.Error()),
            )
            return
        }
    }
    
    // Lookup by name
    if hasName {
        name := config.Name.ValueString()
        tflog.Debug(ctx, "Looking up cache by name", map[string]any{"name": name})
        
        caches, err := d.client.ListCaches(ctx)
        if err != nil {
            resp.Diagnostics.AddError("Query Failed", err.Error())
            return
        }
        
        // Find cache with matching name
        var matching []client.Cache
        for _, c := range caches {
            if c.Name == name {
                matching = append(matching, c)
            }
        }
        
        if len(matching) == 0 {
            resp.Diagnostics.AddError(
                "Cache Not Found",
                fmt.Sprintf("No cache found with name: %s", name),
            )
            return
        }
        
        if len(matching) > 1 {
            resp.Diagnostics.AddError(
                "Multiple Caches Found",
                fmt.Sprintf("Found %d caches with name '%s'. Name must be unique", len(matching), name),
            )
            return
        }
        
        c := matching[0]
        cache = &c
    }
    
    // Lookup by tags
    if hasTags {
        var filterTags map[string]string
        config.Tags.ElementsAs(ctx, &filterTags, false)
        
        tflog.Debug(ctx, "Looking up cache by tags", map[string]any{"tags": filterTags})
        
        caches, err := d.client.ListCaches(ctx)
        if err != nil {
            resp.Diagnostics.AddError("Query Failed", err.Error())
            return
        }
        
        // Filter by tags
        var matching []client.Cache
        for _, c := range caches {
            if tagsMatch(c.Tags, filterTags) {
                matching = append(matching, c)
            }
        }
        
        if len(matching) == 0 {
            resp.Diagnostics.AddError(
                "No Cache Found",
                fmt.Sprintf("No cache found with tags: %v", filterTags),
            )
            return
        }
        
        if len(matching) > 1 {
            resp.Diagnostics.AddError(
                "Multiple Caches Found",
                fmt.Sprintf("Found %d caches with specified tags", len(matching)),
            )
            return
        }
        
        c := matching[0]
        cache = &c
    }
    
    // Populate all attributes
    config.ID = types.StringValue(cache.ID)
    config.Name = types.StringValue(cache.Name)
    config.Engine = types.StringValue(cache.Engine)
    config.EngineVersion = types.StringValue(cache.EngineVersion)
    config.NodeType = types.StringValue(cache.NodeType)
    config.NumNodes = types.Int64Value(int64(cache.NumNodes))
    config.Port = types.Int64Value(int64(cache.Port))
    config.Status = types.StringValue(cache.Status)
    config.Endpoint = types.StringValue(cache.Endpoint)
    config.ARN = types.StringValue(cache.ARN)
    
    // Convert tags
    if len(cache.Tags) > 0 {
        tagsMap := make(map[string]attr.Value)
        for k, v := range cache.Tags {
            tagsMap[k] = types.StringValue(v)
        }
        mapValue, _ := types.MapValue(types.StringType, tagsMap)
        config.Tags = mapValue
    } else {
        config.Tags = types.MapNull(types.StringType)
    }
    
    // Save state
    resp.State.Set(ctx, config)
    
    tflog.Info(ctx, "Successfully read cache", map[string]any{
        "id": cache.ID,
    })
}

func tagsMatch(cacheTags, filterTags map[string]string) bool {
    for key, value := range filterTags {
        cacheValue, exists := cacheTags[key]
        if !exists || cacheValue != value {
            return false
        }
    }
    return true
}

Step 2: List Caches Data Source

Create internal/provider/caches_data_source.go:

package provider

import (
    "context"
    
    "github.com/hashicorp/terraform-plugin-framework/attr"
    "github.com/hashicorp/terraform-plugin-framework/datasource"
    "github.com/hashicorp/terraform-plugin-framework/datasource/schema"
    "github.com/hashicorp/terraform-plugin-framework/types"
    "github.com/hashicorp/terraform-plugin-log/tflog"
    
    "github.com/yourorg/terraform-provider-cachecloud/internal/client"
)

type CachesDataSource struct {
    client *client.Client
}

type CachesDataSourceModel struct {
    IDs     types.List    `tfsdk:"ids"`
    Caches  types.List    `tfsdk:"caches"`
    Filters []FilterModel `tfsdk:"filter"`
}

type FilterModel struct {
    Name   types.String `tfsdk:"name"`
    Values types.List   `tfsdk:"values"`
}

var _ datasource.DataSource = &CachesDataSource{}
var _ datasource.DataSourceWithConfigure = &CachesDataSource{}

func NewCachesDataSource() datasource.DataSource {
    return &CachesDataSource{}
}

func (d *CachesDataSource) Metadata(ctx context.Context, req datasource.MetadataRequest, resp *datasource.MetadataResponse) {
    resp.TypeName = req.ProviderTypeName + "_caches"
}

func (d *CachesDataSource) Schema(ctx context.Context, req datasource.SchemaRequest, resp *datasource.SchemaResponse) {
    resp.Schema = schema.Schema{
        MarkdownDescription: "Fetches a list of cache clusters matching filters",
        
        Attributes: map[string]schema.Attribute{
            "ids": schema.ListAttribute{
                Computed:            true,
                ElementType:         types.StringType,
                MarkdownDescription: "List of cache cluster IDs",
            },
            
            "caches": schema.ListNestedAttribute{
                Computed:            true,
                MarkdownDescription: "Detailed cache cluster information",
                
                NestedObject: schema.NestedAttributeObject{
                    Attributes: map[string]schema.Attribute{
                        "id": schema.StringAttribute{
                            Computed: true,
                        },
                        "name": schema.StringAttribute{
                            Computed: true,
                        },
                        "engine": schema.StringAttribute{
                            Computed: true,
                        },
                        "node_type": schema.StringAttribute{
                            Computed: true,
                        },
                        "num_nodes": schema.Int64Attribute{
                            Computed: true,
                        },
                        "status": schema.StringAttribute{
                            Computed: true,
                        },
                        "endpoint": schema.StringAttribute{
                            Computed: true,
                        },
                    },
                },
            },
        },
        
        Blocks: map[string]schema.Block{
            "filter": schema.ListNestedBlock{
                MarkdownDescription: "Filter caches. Multiple filters are ANDed",
                
                NestedObject: schema.NestedBlockObject{
                    Attributes: map[string]schema.Attribute{
                        "name": schema.StringAttribute{
                            Required:            true,
                            MarkdownDescription: "Filter name (tag:<key>, status, engine)",
                        },
                        "values": schema.ListAttribute{
                            Required:            true,
                            ElementType:         types.StringType,
                            MarkdownDescription: "Filter values (OR within filter)",
                        },
                    },
                },
            },
        },
    }
}

func (d *CachesDataSource) Configure(ctx context.Context, req datasource.ConfigureRequest, resp *datasource.ConfigureResponse) {
    if req.ProviderData == nil {
        return
    }
    
    client, ok := req.ProviderData.(*client.Client)
    if !ok {
        resp.Diagnostics.AddError(
            "Unexpected Data Source Configure Type",
            fmt.Sprintf("Expected *client.Client, got: %T", req.ProviderData),
        )
        return
    }
    
    d.client = client
}

func (d *CachesDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config CachesDataSourceModel
    req.Config.Get(ctx, &config)
    
    tflog.Debug(ctx, "Reading caches data source")
    
    // Get all caches
    caches, err := d.client.ListCaches(ctx)
    if err != nil {
        resp.Diagnostics.AddError("Query Failed", err.Error())
        return
    }
    
    tflog.Debug(ctx, "Retrieved caches", map[string]any{
        "count": len(caches),
    })
    
    // Apply filters
    filtered := caches
    
    for _, filter := range config.Filters {
        name := filter.Name.ValueString()
        
        var values []string
        filter.Values.ElementsAs(ctx, &values, false)
        
        filtered = applyFilter(filtered, name, values)
        
        tflog.Debug(ctx, "Applied filter", map[string]any{
            "name":      name,
            "values":    values,
            "remaining": len(filtered),
        })
    }
    
    // Handle empty results
    if len(filtered) == 0 {
        tflog.Info(ctx, "No caches found matching criteria")
        config.IDs = types.ListNull(types.StringType)
        config.Caches = types.ListNull(cacheObjectType)
        resp.State.Set(ctx, config)
        return
    }
    
    // Build ID list
    ids := make([]attr.Value, len(filtered))
    for i, cache := range filtered {
        ids[i] = types.StringValue(cache.ID)
    }
    
    idsList, _ := types.ListValue(types.StringType, ids)
    config.IDs = idsList
    
    // Build detailed list
    cacheObjectType := types.ObjectType{
        AttrTypes: map[string]attr.Type{
            "id":        types.StringType,
            "name":      types.StringType,
            "engine":    types.StringType,
            "node_type": types.StringType,
            "num_nodes": types.Int64Type,
            "status":    types.StringType,
            "endpoint":  types.StringType,
        },
    }
    
    cacheObjects := make([]attr.Value, len(filtered))
    
    for i, cache := range filtered {
        objValue, _ := types.ObjectValue(
            cacheObjectType.AttrTypes,
            map[string]attr.Value{
                "id":        types.StringValue(cache.ID),
                "name":      types.StringValue(cache.Name),
                "engine":    types.StringValue(cache.Engine),
                "node_type": types.StringValue(cache.NodeType),
                "num_nodes": types.Int64Value(int64(cache.NumNodes)),
                "status":    types.StringValue(cache.Status),
                "endpoint":  types.StringValue(cache.Endpoint),
            },
        )
        cacheObjects[i] = objValue
    }
    
    cachesList, _ := types.ListValue(cacheObjectType, cacheObjects)
    config.Caches = cachesList
    
    // Save state
    resp.State.Set(ctx, config)
    
    tflog.Info(ctx, "Successfully read caches", map[string]any{
        "count": len(filtered),
    })
}

func applyFilter(caches []client.Cache, filterName string, filterValues []string) []client.Cache {
    var result []client.Cache
    
    for _, cache := range caches {
        if matchesFilter(cache, filterName, filterValues) {
            result = append(result, cache)
        }
    }
    
    return result
}

func matchesFilter(cache client.Cache, name string, values []string) bool {
    // Tag filter
    if strings.HasPrefix(name, "tag:") {
        tagKey := strings.TrimPrefix(name, "tag:")
        tagValue, exists := cache.Tags[tagKey]
        if !exists {
            return false
        }
        
        for _, value := range values {
            if tagValue == value {
                return true
            }
        }
        return false
    }
    
    // Status filter
    if name == "status" {
        for _, value := range values {
            if cache.Status == value {
                return true
            }
        }
        return false
    }
    
    // Engine filter
    if name == "engine" {
        for _, value := range values {
            if cache.Engine == value {
                return true
            }
        }
        return false
    }
    
    return false
}

Step 3: Update Client

Add ListCaches to internal/client/client.go:

func (c *Client) ListCaches(ctx context.Context) ([]Cache, error) {
    // Simulate API call
    fmt.Println("Listing all caches")
    
    caches := []Cache{
        {
            ID:            "cache-1",
            Name:          "prod-redis",
            Engine:        "redis",
            EngineVersion: "7.0",
            NodeType:      "cache.t3.micro",
            NumNodes:      2,
            Port:          6379,
            Status:        "available",
            Endpoint:      "prod-redis.cache.example.com:6379",
            ARN:           "arn:aws:cache:us-east-1:123456789012:cache/prod-redis",
            Tags:          map[string]string{"Environment": "production", "Type": "redis"},
        },
        {
            ID:            "cache-2",
            Name:          "dev-redis",
            Engine:        "redis",
            EngineVersion: "7.0",
            NodeType:      "cache.t3.micro",
            NumNodes:      1,
            Port:          6379,
            Status:        "available",
            Endpoint:      "dev-redis.cache.example.com:6379",
            ARN:           "arn:aws:cache:us-east-1:123456789012:cache/dev-redis",
            Tags:          map[string]string{"Environment": "development", "Type": "redis"},
        },
        {
            ID:            "cache-3",
            Name:          "prod-memcached",
            Engine:        "memcached",
            EngineVersion: "1.6",
            NodeType:      "cache.t3.small",
            NumNodes:      2,
            Port:          11211,
            Status:        "available",
            Endpoint:      "prod-memcached.cache.example.com:11211",
            ARN:           "arn:aws:cache:us-east-1:123456789012:cache/prod-memcached",
            Tags:          map[string]string{"Environment": "production", "Type": "memcached"},
        },
    }
    
    return caches, nil
}

Step 4: Register Data Sources

Update internal/provider/provider.go:

func (p *CacheCloudProvider) DataSources(ctx context.Context) []func() datasource.DataSource {
    return []func() datasource.DataSource{
        NewCacheDataSource,
        NewCachesDataSource,
    }
}

Step 5: Build Provider

go mod tidy
go build -o terraform-provider-cachecloud

Install locally:

# macOS ARM64
mkdir -p ~/.terraform.d/plugins/registry.terraform.io/yourorg/cachecloud/1.0.0/darwin_arm64
cp terraform-provider-cachecloud ~/.terraform.d/plugins/registry.terraform.io/yourorg/cachecloud/1.0.0/darwin_arm64/

Step 6: Test Single Cache Lookup

Create examples/data-sources/cache.tf:

terraform {
  required_providers {
    cachecloud = {
      source = "registry.terraform.io/yourorg/cachecloud"
    }
  }
}

provider "cachecloud" {
  api_key = "test-key"
}

# Lookup by ID
data "cachecloud_cache" "by_id" {
  id = "cache-1"
}

# Lookup by name
data "cachecloud_cache" "by_name" {
  name = "prod-redis"
}

# Lookup by tags
data "cachecloud_cache" "by_tags" {
  tags = {
    Environment = "production"
    Type        = "redis"
  }
}

# Outputs
output "cache_by_id" {
  value = {
    id       = data.cachecloud_cache.by_id.id
    name     = data.cachecloud_cache.by_id.name
    endpoint = data.cachecloud_cache.by_id.endpoint
  }
}

output "cache_by_name" {
  value = {
    id       = data.cachecloud_cache.by_name.id
    endpoint = data.cachecloud_cache.by_name.endpoint
  }
}

output "cache_by_tags" {
  value = {
    id       = data.cachecloud_cache.by_tags.id
    name     = data.cachecloud_cache.by_tags.name
  }
}

Test:

cd examples/data-sources
terraform init
terraform plan
terraform apply

Step 7: Test List Data Source

Create examples/data-sources/caches.tf:

# Get all production caches
data "cachecloud_caches" "production" {
  filter {
    name   = "tag:Environment"
    values = ["production"]
  }
}

# Get all Redis caches
data "cachecloud_caches" "redis" {
  filter {
    name   = "engine"
    values = ["redis"]
  }
}

# Get available production Redis caches
data "cachecloud_caches" "prod_redis" {
  filter {
    name   = "tag:Environment"
    values = ["production"]
  }
  
  filter {
    name   = "engine"
    values = ["redis"]
  }
  
  filter {
    name   = "status"
    values = ["available"]
  }
}

# Outputs
output "production_cache_ids" {
  value = data.cachecloud_caches.production.ids
}

output "redis_caches" {
  value = data.cachecloud_caches.redis.caches
}

output "prod_redis_count" {
  value = length(data.cachecloud_caches.prod_redis.ids)
}

# Use in resources
resource "cachecloud_backup" "production" {
  for_each = toset(data.cachecloud_caches.production.ids)
  
  cache_id = each.value
  schedule = "daily"
}

Test:

terraform plan
terraform apply

Lab Validation

Test Checklist

  • Single cache lookup by ID works
  • Single cache lookup by name works
  • Single cache lookup by tags works
  • Error when multiple caches match (name)
  • Error when no cache found
  • List data source returns all caches
  • Tag filtering works correctly
  • Engine filtering works correctly
  • Multiple filters combine with AND logic
  • Empty filter results return empty list
  • Can use data source IDs in for_each

Expected Output

terraform apply

data.cachecloud_cache.by_id: Reading...
data.cachecloud_cache.by_name: Reading...
data.cachecloud_cache.by_tags: Reading...
data.cachecloud_caches.production: Reading...
data.cachecloud_caches.redis: Reading...
data.cachecloud_caches.prod_redis: Reading...

data.cachecloud_cache.by_id: Read complete after 0s
data.cachecloud_cache.by_name: Read complete after 0s
data.cachecloud_cache.by_tags: Read complete after 0s
data.cachecloud_caches.production: Read complete after 0s [id=2]
data.cachecloud_caches.redis: Read complete after 0s [id=2]
data.cachecloud_caches.prod_redis: Read complete after 0s [id=1]

Changes to Outputs:
  + cache_by_id         = {
      + endpoint = "prod-redis.cache.example.com:6379"
      + id       = "cache-1"
      + name     = "prod-redis"
    }
  + production_cache_ids = [
      + "cache-1",
      + "cache-3",
    ]
  + prod_redis_count    = 1

Challenge Exercises

1. Add Most Recent

Add a most_recent attribute to return the newest cache when multiple match.

2. Add Name Pattern

Support wildcard name matching (e.g., prod-*).

3. Add Sorting

Add sort_by and sort_order attributes to the list data source.

4. Add Validation

Validate filter names and provide helpful error messages for unknown filters.

5. Add Region Filter

Add a region filter to the list data source.

Key Takeaways

  • ✅ Data sources query existing infrastructure
  • ✅ Single data sources return exactly one resource
  • ✅ List data sources return zero or more resources
  • ✅ Support multiple lookup methods (ID, name, tags)
  • ✅ Implement filter blocks for flexible queries
  • ✅ Filters combine with AND logic between blocks
  • ✅ Values within a filter use OR logic
  • ✅ Return empty lists, not errors, for no matches
  • ✅ Provide detailed error messages
  • ✅ Log query operations for debugging

What’s Next?

Test your knowledge with comprehensive practice questions!

Next: Data Sources Practice →

Previous: ← Data Source Lists

Last updated: June 30, 2026

Updated:

You may also enjoy