Data Source Filtering

11 minute read

Why Filtering Matters

Filtering enables users to query infrastructure using flexible criteria without knowing exact IDs or names. This is essential for dynamic configurations and discovering resources.

Common filtering needs:

  • Find resources by tags
  • Filter by status or state
  • Match name patterns
  • Query by region or availability zone
  • Filter by creation time
  • Combine multiple criteria

Filter Block Pattern

Schema Definition

func (d *InstancesDataSource) Schema(ctx context.Context, req datasource.SchemaRequest, resp *datasource.SchemaResponse) {
    resp.Schema = schema.Schema{
        MarkdownDescription: "Fetches a list of compute instances matching filters",
        
        Attributes: map[string]schema.Attribute{
            "ids": schema.ListAttribute{
                Computed:            true,
                ElementType:         types.StringType,
                MarkdownDescription: "List of instance IDs matching filters",
            },
        },
        
        Blocks: map[string]schema.Block{
            "filter": schema.ListNestedBlock{
                MarkdownDescription: "Filter instances. Multiple filters are ANDed together",
                
                NestedObject: schema.NestedBlockObject{
                    Attributes: map[string]schema.Attribute{
                        "name": schema.StringAttribute{
                            Required:            true,
                            MarkdownDescription: "Name of the filter field (e.g., 'tag:Name', 'instance-state-name')",
                        },
                        "values": schema.ListAttribute{
                            Required:            true,
                            ElementType:         types.StringType,
                            MarkdownDescription: "Filter values. Multiple values are ORed within the filter",
                        },
                    },
                },
            },
        },
    }
}

Data Model

type InstancesDataSourceModel struct {
    IDs     types.List    `tfsdk:"ids"`
    Filters []FilterModel `tfsdk:"filter"`
}

type FilterModel struct {
    Name   types.String `tfsdk:"name"`
    Values types.List   `tfsdk:"values"`
}

Usage Example

data "mycloud_instances" "web_servers" {
  # Find instances with tag Role=web OR Role=frontend
  filter {
    name   = "tag:Role"
    values = ["web", "frontend"]
  }
  
  # AND tag Environment=production
  filter {
    name   = "tag:Environment"
    values = ["production"]
  }
  
  # AND state is running
  filter {
    name   = "instance-state-name"
    values = ["running"]
  }
}

output "instance_ids" {
  value = data.mycloud_instances.web_servers.ids
}

Implementing Filter Logic

Parsing Filters

func (d *InstancesDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config InstancesDataSourceModel
    req.Config.Get(ctx, &config)
    
    // Convert filters to usable format
    filters := make(map[string][]string)
    
    for _, filter := range config.Filters {
        name := filter.Name.ValueString()
        
        var values []string
        diags := filter.Values.ElementsAs(ctx, &values, false)
        resp.Diagnostics.Append(diags...)
        if resp.Diagnostics.HasError() {
            return
        }
        
        filters[name] = values
        
        tflog.Debug(ctx, "Processing filter", map[string]any{
            "name":   name,
            "values": values,
        })
    }
    
    // Query API with filters
    instances, err := d.client.ListInstances(ctx, filters)
    if err != nil {
        resp.Diagnostics.AddError("Query Failed", err.Error())
        return
    }
    
    // Extract IDs
    ids := make([]attr.Value, len(instances))
    for i, instance := range instances {
        ids[i] = types.StringValue(instance.ID)
    }
    
    listValue, diags := types.ListValue(types.StringType, ids)
    resp.Diagnostics.Append(diags...)
    config.IDs = listValue
    
    resp.State.Set(ctx, config)
}

Client-Side Filtering

If your API doesn’t support filtering, implement it client-side:

func (d *InstancesDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config InstancesDataSourceModel
    req.Config.Get(ctx, &config)
    
    // Get all instances
    allInstances, err := d.client.ListAllInstances(ctx)
    if err != nil {
        resp.Diagnostics.AddError("Query Failed", err.Error())
        return
    }
    
    tflog.Debug(ctx, "Retrieved instances", map[string]any{
        "count": len(allInstances),
    })
    
    // Apply filters
    filtered := allInstances
    
    for _, filter := range config.Filters {
        name := filter.Name.ValueString()
        
        var values []string
        filter.Values.ElementsAs(ctx, &values, false)
        
        filtered = applyFilter(filtered, name, values)
        
        tflog.Debug(ctx, "Applied filter", map[string]any{
            "name":      name,
            "values":    values,
            "remaining": len(filtered),
        })
    }
    
    // Convert to IDs
    ids := make([]attr.Value, len(filtered))
    for i, instance := range filtered {
        ids[i] = types.StringValue(instance.ID)
    }
    
    listValue, _ := types.ListValue(types.StringType, ids)
    config.IDs = listValue
    
    resp.State.Set(ctx, config)
}

func applyFilter(instances []Instance, filterName string, filterValues []string) []Instance {
    var result []Instance
    
    for _, instance := range instances {
        if matchesFilter(instance, filterName, filterValues) {
            result = append(result, instance)
        }
    }
    
    return result
}

func matchesFilter(instance Instance, name string, values []string) bool {
    // Tag filters
    if strings.HasPrefix(name, "tag:") {
        tagKey := strings.TrimPrefix(name, "tag:")
        tagValue, exists := instance.Tags[tagKey]
        if !exists {
            return false
        }
        
        // Check if tag value matches any filter value (OR logic)
        for _, value := range values {
            if tagValue == value {
                return true
            }
        }
        return false
    }
    
    // State filter
    if name == "instance-state-name" {
        for _, value := range values {
            if instance.State == value {
                return true
            }
        }
        return false
    }
    
    // Type filter
    if name == "instance-type" {
        for _, value := range values {
            if instance.Type == value {
                return true
            }
        }
        return false
    }
    
    // Add more filter types as needed
    
    return false
}

Tag Filtering

Exact Tag Match

func (d *DatabaseDataSource) Schema(ctx context.Context, req datasource.SchemaRequest, resp *datasource.SchemaResponse) {
    resp.Schema = schema.Schema{
        Attributes: map[string]schema.Attribute{
            "tags": schema.MapAttribute{
                Optional:            true,
                ElementType:         types.StringType,
                MarkdownDescription: "Tags to filter by. All specified tags must match exactly",
            },
            
            "id": schema.StringAttribute{
                Computed: true,
            },
            "name": schema.StringAttribute{
                Computed: true,
            },
        },
    }
}

func (d *DatabaseDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config DatabaseDataSourceModel
    req.Config.Get(ctx, &config)
    
    // Parse tags
    var filterTags map[string]string
    config.Tags.ElementsAs(ctx, &filterTags, false)
    
    tflog.Debug(ctx, "Filtering by tags", map[string]any{
        "tags": filterTags,
    })
    
    // Get all databases
    allDbs, err := d.client.ListDatabases(ctx)
    if err != nil {
        resp.Diagnostics.AddError("Query Failed", err.Error())
        return
    }
    
    // Filter by tags (all must match)
    var matchingDbs []Database
    
    for _, db := range allDbs {
        if tagsMatch(db.Tags, filterTags) {
            matchingDbs = append(matchingDbs, db)
        }
    }
    
    if len(matchingDbs) == 0 {
        resp.Diagnostics.AddError(
            "No Database Found",
            fmt.Sprintf("No database found with tags: %v", filterTags),
        )
        return
    }
    
    if len(matchingDbs) > 1 {
        resp.Diagnostics.AddError(
            "Multiple Databases Found",
            fmt.Sprintf("Found %d databases with specified tags", len(matchingDbs)),
        )
        return
    }
    
    db := matchingDbs[0]
    config.ID = types.StringValue(db.ID)
    config.Name = types.StringValue(db.Name)
    
    resp.State.Set(ctx, config)
}

func tagsMatch(dbTags, filterTags map[string]string) bool {
    // All filter tags must exist and match
    for key, value := range filterTags {
        dbValue, exists := dbTags[key]
        if !exists || dbValue != value {
            return false
        }
    }
    return true
}

Usage:

data "mycloud_database" "shared" {
  tags = {
    Environment = "production"
    Shared      = "true"
    Team        = "platform"
  }
}

Tag Key Existence

func (d *InstancesDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config InstancesDataSourceModel
    req.Config.Get(ctx, &config)
    
    // Filter instances that have specific tags (regardless of value)
    requiredTagKeys := config.RequiredTags.Elements()
    
    var filtered []Instance
    for _, instance := range allInstances {
        hasAllTags := true
        
        for _, tagKey := range requiredTagKeys {
            key := tagKey.(types.String).ValueString()
            if _, exists := instance.Tags[key]; !exists {
                hasAllTags = false
                break
            }
        }
        
        if hasAllTags {
            filtered = append(filtered, instance)
        }
    }
    
    // Set results...
}

Name Pattern Filtering

Wildcard Matching

"name_pattern": schema.StringAttribute{
    Optional:            true,
    MarkdownDescription: "Name pattern with wildcards (* matches any characters)",
},

func (d *AMIDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config AMIDataSourceModel
    req.Config.Get(ctx, &config)
    
    pattern := config.NamePattern.ValueString()
    
    // Convert wildcard pattern to regex
    regexPattern := "^" + strings.ReplaceAll(regexp.QuoteMeta(pattern), `\*`, ".*") + "$"
    nameRegex, err := regexp.Compile(regexPattern)
    if err != nil {
        resp.Diagnostics.AddError(
            "Invalid Pattern",
            fmt.Sprintf("Invalid name pattern: %s", err.Error()),
        )
        return
    }
    
    // Query and filter AMIs
    amis, err := d.client.ListAMIs(ctx)
    if err != nil {
        resp.Diagnostics.AddError("Query Failed", err.Error())
        return
    }
    
    var matching []AMI
    for _, ami := range amis {
        if nameRegex.MatchString(ami.Name) {
            matching = append(matching, ami)
        }
    }
    
    if len(matching) == 0 {
        resp.Diagnostics.AddError(
            "No AMI Found",
            fmt.Sprintf("No AMI found matching pattern: %s", pattern),
        )
        return
    }
    
    // Use most recent if multiple matches
    ami := findMostRecent(matching)
    
    config.ID = types.StringValue(ami.ID)
    config.Name = types.StringValue(ami.Name)
    
    resp.State.Set(ctx, config)
}

Usage:

data "mycloud_ami" "ubuntu" {
  name_pattern = "ubuntu-22.04-*-amd64-server-*"
  most_recent  = true
}

Regex Matching

"name_regex": schema.StringAttribute{
    Optional:            true,
    MarkdownDescription: "Regular expression to match AMI names",
},

func (d *AMIDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config AMIDataSourceModel
    req.Config.Get(ctx, &config)
    
    pattern := config.NameRegex.ValueString()
    
    nameRegex, err := regexp.Compile(pattern)
    if err != nil {
        resp.Diagnostics.AddError(
            "Invalid Regex",
            fmt.Sprintf("Invalid regular expression: %s", err.Error()),
        )
        return
    }
    
    amis, err := d.client.ListAMIs(ctx)
    if err != nil {
        resp.Diagnostics.AddError("Query Failed", err.Error())
        return
    }
    
    var matching []AMI
    for _, ami := range amis {
        if nameRegex.MatchString(ami.Name) {
            matching = append(matching, ami)
        }
    }
    
    // Handle results...
}

Usage:

data "mycloud_ami" "ubuntu" {
  name_regex  = "^ubuntu-22\\.04-\\d{8}-amd64-server-.*$"
  most_recent = true
}

Most Recent Pattern

Schema

"most_recent": schema.BoolAttribute{
    Optional:            true,
    MarkdownDescription: "If multiple results match, return the most recent. Defaults to false",
},

"created_at": schema.StringAttribute{
    Computed:            true,
    MarkdownDescription: "Creation timestamp of the selected resource",
},

Implementation

func (d *AMIDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config AMIDataSourceModel
    req.Config.Get(ctx, &config)
    
    // Apply filters to get matching AMIs
    matching := filterAMIs(allAMIs, config)
    
    if len(matching) == 0 {
        resp.Diagnostics.AddError(
            "No AMI Found",
            "No AMI found matching the specified criteria",
        )
        return
    }
    
    var ami AMI
    
    if config.MostRecent.ValueBool() {
        // Sort by creation time and take most recent
        ami = findMostRecent(matching)
        
        tflog.Info(ctx, "Selected most recent AMI", map[string]any{
            "id":         ami.ID,
            "name":       ami.Name,
            "created_at": ami.CreatedAt,
            "total":      len(matching),
        })
    } else {
        // Require exactly one match
        if len(matching) > 1 {
            resp.Diagnostics.AddError(
                "Multiple AMIs Found",
                fmt.Sprintf(
                    "Found %d AMIs matching criteria. "+
                    "Set most_recent = true or refine filters.\n\n"+
                    "Matching AMIs:\n%s",
                    len(matching),
                    formatAMIList(matching),
                ),
            )
            return
        }
        
        ami = matching[0]
    }
    
    // Set computed attributes
    config.ID = types.StringValue(ami.ID)
    config.Name = types.StringValue(ami.Name)
    config.CreatedAt = types.StringValue(ami.CreatedAt)
    
    resp.State.Set(ctx, config)
}

func findMostRecent(amis []AMI) AMI {
    if len(amis) == 0 {
        return AMI{}
    }
    
    mostRecent := amis[0]
    
    for _, ami := range amis[1:] {
        if ami.CreatedAt > mostRecent.CreatedAt {
            mostRecent = ami
        }
    }
    
    return mostRecent
}

Usage:

data "mycloud_ami" "latest_ubuntu" {
  most_recent = true
  
  filter {
    name   = "name"
    values = ["ubuntu-22.04-*"]
  }
  
  filter {
    name   = "architecture"
    values = ["x86_64"]
  }
}

output "ami_id" {
  value = data.mycloud_ami.latest_ubuntu.id
}

output "ami_created" {
  value = data.mycloud_ami.latest_ubuntu.created_at
}

Complex Filter Combinations

AND/OR Logic

// Multiple filters are ANDed
data "mycloud_instances" "specific" {
  filter {
    name   = "tag:Environment"
    values = ["production"]  # Must match
  }
  
  filter {
    name   = "instance-state-name"
    values = ["running", "pending"]  # Must match one of these (OR)
  }
  
  filter {
    name   = "instance-type"
    values = ["t3.large", "t3.xlarge"]  # Must match one of these (OR)
  }
}

Implementation

func (d *InstancesDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config InstancesDataSourceModel
    req.Config.Get(ctx, &config)
    
    instances, err := d.client.ListInstances(ctx)
    if err != nil {
        resp.Diagnostics.AddError("Query Failed", err.Error())
        return
    }
    
    // Apply each filter (AND logic between filters)
    filtered := instances
    
    for _, filter := range config.Filters {
        name := filter.Name.ValueString()
        
        var values []string
        filter.Values.ElementsAs(ctx, &values, false)
        
        // Filter must match at least one value (OR logic within filter)
        filtered = filterInstances(filtered, name, values)
    }
    
    // Convert to results
    ids := make([]attr.Value, len(filtered))
    for i, instance := range filtered {
        ids[i] = types.StringValue(instance.ID)
    }
    
    listValue, _ := types.ListValue(types.StringType, ids)
    config.IDs = listValue
    
    resp.State.Set(ctx, config)
}

func filterInstances(instances []Instance, filterName string, filterValues []string) []Instance {
    var result []Instance
    
    for _, instance := range instances {
        // Check if instance matches any value (OR logic)
        matches := false
        
        for _, value := range filterValues {
            if instanceMatchesFilterValue(instance, filterName, value) {
                matches = true
                break
            }
        }
        
        if matches {
            result = append(result, instance)
        }
    }
    
    return result
}

State/Status Filtering

"state": schema.StringAttribute{
    Optional:            true,
    MarkdownDescription: "Filter by state (available, creating, deleting, etc.)",
},

func (d *DatabasesDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config DatabasesDataSourceModel
    req.Config.Get(ctx, &config)
    
    dbs, err := d.client.ListDatabases(ctx)
    if err != nil {
        resp.Diagnostics.AddError("Query Failed", err.Error())
        return
    }
    
    // Filter by state if specified
    if !config.State.IsNull() {
        state := config.State.ValueString()
        
        var filtered []Database
        for _, db := range dbs {
            if db.Status == state {
                filtered = append(filtered, db)
            }
        }
        
        dbs = filtered
        
        tflog.Debug(ctx, "Filtered by state", map[string]any{
            "state": state,
            "count": len(dbs),
        })
    }
    
    // Convert to IDs
    ids := make([]attr.Value, len(dbs))
    for i, db := range dbs {
        ids[i] = types.StringValue(db.ID)
    }
    
    listValue, _ := types.ListValue(types.StringType, ids)
    config.IDs = listValue
    
    resp.State.Set(ctx, config)
}

Usage:

data "mycloud_databases" "available" {
  state = "available"
  
  tags = {
    Environment = "production"
  }
}

Sorting Results

"sort_by": schema.StringAttribute{
    Optional:            true,
    MarkdownDescription: "Sort results by field (name, created_at, etc.)",
},

"sort_order": schema.StringAttribute{
    Optional:            true,
    MarkdownDescription: "Sort order (asc or desc). Defaults to asc",
},

func (d *DatabasesDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) {
    var config DatabasesDataSourceModel
    req.Config.Get(ctx, &config)
    
    // Query and filter databases
    dbs := queryAndFilter(d.client, config)
    
    // Sort if requested
    if !config.SortBy.IsNull() {
        sortBy := config.SortBy.ValueString()
        sortOrder := "asc"
        
        if !config.SortOrder.IsNull() {
            sortOrder = config.SortOrder.ValueString()
        }
        
        sortDatabases(dbs, sortBy, sortOrder)
    }
    
    // Convert to results...
}

func sortDatabases(dbs []Database, sortBy, order string) {
    sort.Slice(dbs, func(i, j int) bool {
        var less bool
        
        switch sortBy {
        case "name":
            less = dbs[i].Name < dbs[j].Name
        case "created_at":
            less = dbs[i].CreatedAt < dbs[j].CreatedAt
        case "storage_gb":
            less = dbs[i].StorageGB < dbs[j].StorageGB
        default:
            less = dbs[i].Name < dbs[j].Name
        }
        
        if order == "desc" {
            return !less
        }
        return less
    })
}

Validation

Filter Name Validation

func (d *InstancesDataSource) ValidateConfig(ctx context.Context, req datasource.ValidateConfigRequest, resp *datasource.ValidateConfigResponse) {
    var config InstancesDataSourceModel
    req.Config.Get(ctx, &config)
    
    validFilterNames := []string{
        "instance-state-name",
        "instance-type",
        "availability-zone",
        "tag:Name",
        "tag:Environment",
        "tag:Role",
    }
    
    for _, filter := range config.Filters {
        name := filter.Name.ValueString()
        
        // Check if it's a valid filter or tag filter
        if !isValidFilterName(name, validFilterNames) {
            resp.Diagnostics.AddAttributeWarning(
                path.Root("filter"),
                "Unknown Filter Name",
                fmt.Sprintf(
                    "Filter name '%s' may not be supported.\n\n"+
                    "Valid filter names:\n%s\n\n"+
                    "Tag filters use format: tag:<TagKey>",
                    name,
                    strings.Join(validFilterNames, "\n"),
                ),
            )
        }
    }
}

func isValidFilterName(name string, validNames []string) bool {
    // Check exact match
    for _, valid := range validNames {
        if name == valid {
            return true
        }
    }
    
    // Check if it's a tag filter
    if strings.HasPrefix(name, "tag:") {
        return true
    }
    
    return false
}

Best Practices

1. Provide Clear Filter Documentation

resp.Schema = schema.Schema{
    MarkdownDescription: "Fetches instances matching filters.\n\n" +
        "## Supported Filters\n\n" +
        "- `instance-state-name`: running, stopped, pending, etc.\n" +
        "- `instance-type`: t3.micro, t3.small, etc.\n" +
        "- `availability-zone`: us-east-1a, us-east-1b, etc.\n" +
        "- `tag:<TagKey>`: Filter by tag key and value\n\n" +
        "## Example\n\n" +
        "```hcl\n" +
        "data \"mycloud_instances\" \"web\" {\n" +
        "  filter {\n" +
        "    name   = \"tag:Role\"\n" +
        "    values = [\"web\"]\n" +
        "  }\n" +
        "}\n" +
        "```",
}

2. Log Filter Application

tflog.Debug(ctx, "Applying filter", map[string]any{
    "name":            name,
    "values":          values,
    "before_count":    len(instances),
    "after_count":     len(filtered),
})

3. Handle Empty Results

if len(filtered) == 0 {
    resp.Diagnostics.AddError(
        "No Resources Found",
        fmt.Sprintf(
            "No resources found matching filters.\n\n"+
            "Filters applied:\n%s\n\n"+
            "Check filter names and values are correct.",
            formatFilters(config.Filters),
        ),
    )
    return
}

4. Optimize Performance

// ✅ Good: Filter on server side
instances, err := d.client.ListInstances(ctx, &ListOptions{
    Filters: convertFilters(config.Filters),
})

// ❌ Bad: Fetch all, filter client side
all, err := d.client.ListAllInstances(ctx)
filtered := filterClientSide(all, config.Filters)

Key Takeaways

  • ✅ Use filter blocks for flexible querying
  • ✅ Support multiple filter combinations (AND between filters, OR within)
  • ✅ Implement tag filtering for resource discovery
  • ✅ Support name pattern matching (wildcards or regex)
  • ✅ Provide “most_recent” option for time-based selection
  • ✅ Validate filter names and provide helpful errors
  • ✅ Log filter application for debugging
  • ✅ Handle empty results gracefully
  • ✅ Optimize by filtering server-side when possible
  • ✅ Document supported filters clearly

What’s Next?

Learn how to handle data sources that return lists of resources!

Next: Handling Lists →

Previous: ← Data Source Read

Last updated: June 30, 2026

Updated:

You may also enjoy