Add exclude_services filter

PROVIDERS.md

@@ -1,5 +1,20 @@
 # Providers
 
+## Service filtering
+
+Every provider enumerates all of its supported services by default. Two optional, additive keys narrow that set (also available on the CLI as `-s`/`--service` and `-es`/`--exclude-service`):
+
+- `services` (list): allowlist. Only the listed services are enumerated. Unknown values are ignored.
+- `exclude_services` (list): blocklist applied after `services` (or after the default-all set when `services` is omitted). Unknown values are ignored.
+
+```yaml
+- provider: gcp
+  exclude_services:
+    - cloud-function
+  project_ids:
+    - my-project
+```
+
 ### Amazon Web Services (AWS)
 
 Amazon Web Services can be integrated by using the following configuration block.

README.md

@@ -64,12 +64,13 @@ CONFIGURATION:
    -pc, -provider-config string  provider config file (default "$HOME/.config/cloudlist/provider-config.yaml")
 
 FILTERS:
-   -p, -provider value    display results for given providers (comma-separated) (default linode,fastly,heroku,terraform,digitalocean,consul,cloudflare,hetzner,nomad,do,scw,openstack,alibaba,aws,gcp,namecheap,kubernetes,azure, custom)
-   -id string[]           display results for given ids (comma-separated)
-   -host                  display only hostnames in results
-   -ip                    display only ips in results
-   -s, -service value     query and display results from given service (comma-separated)) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway)
-   -ep, -exclude-private  exclude private ips in cli output
+   -p, -provider value          display results for given providers (comma-separated) (default linode,fastly,heroku,terraform,digitalocean,consul,cloudflare,hetzner,nomad,do,scw,openstack,alibaba,aws,gcp,namecheap,kubernetes,azure, custom)
+   -id string[]                 display results for given ids (comma-separated)
+   -host                        display only hostnames in results
+   -ip                          display only ips in results
+   -s, -service value           query and display results from given service (comma-separated) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway)
+   -es, -exclude-service value  services to skip for a provider (comma-separated)
+   -ep, -exclude-private        exclude private ips in cli output
 
 UPDATE:
    -up, -update                 update cloudlist to latest version

internal/runner/options.go

@@ -25,13 +25,14 @@ type Options struct {
 	Version            bool                // Version returns the version of the tool.
 	Verbose            bool                // Verbose prints verbose output.
 	Hosts              bool                // Hosts specifies to fetch only DNS Names
-	IPAddress          bool                // IPAddress specifes to fetch only IP Addresses
+	IPAddress          bool                // IPAddress specifies to fetch only IP Addresses
 	Config             string              // Config is the location of the config file.
 	Output             string              // Output is the file to write found results too.
 	ExcludePrivate     bool                // ExcludePrivate excludes private IPs from results
 	Providers          goflags.StringSlice // Providers specifies what providers to fetch assets for.
 	Id                 goflags.StringSlice // Id specifies what id's to fetch assets for.
 	Services           goflags.StringSlice // Services specifies what services to fetch assets for a provider.
+	ExcludeServices    goflags.StringSlice // ExcludeServices specifies what services to skip for a provider.
 	ExtendedMetadata   bool                // ExtendedMetadata enables extended metadata for providers.
 	ProviderConfig     string              // ProviderConfig is the location of the provider config file.
 	DisableUpdateCheck bool                // DisableUpdateCheck disable automatic update check
@@ -40,7 +41,7 @@ type Options struct {
 var (
 	defaultConfigLocation             = filepath.Join(userHomeDir(), ".config/cloudlist/config.yaml")
 	defaultProviderConfigLocation     = filepath.Join(userHomeDir(), ".config/cloudlist/provider-config.yaml")
-	defaultProviders, defaultServies  = []string{}, []string{}
+	defaultProviders, defaultServices = []string{}, []string{}
 	allowedProviders, allowedServices = []string{}, []string{}
 )
 
@@ -51,7 +52,7 @@ func init() {
 	}
 
 	for _, service := range inventory.GetServices() {
-		defaultServies = append(defaultServies, service)
+		defaultServices = append(defaultServices, service)
 		allowedServices = append(allowedServices, service)
 	}
 }
@@ -84,7 +85,8 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.Hosts, "host", false, "display only hostnames in results"),
 		flagSet.BoolVar(&options.IPAddress, "ip", false, "display only ips in results"),
 		flagSet.BoolVar(&options.ExtendedMetadata, "extended-metadata", false, "enable extended metadata for providers"),
-		flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated)) (default "+strings.Join(defaultServies, ",")+")", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated) (default "+strings.Join(defaultServices, ",")+")", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.ExcludeServices, "exclude-service", "es", nil, "services to skip for a provider (comma-separated)", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.ExcludePrivate, "exclude-private", "ep", false, "exclude private ips in cli output"),
 	)
 	flagSet.CreateGroup("update", "Update",

internal/runner/runner.go

@@ -38,10 +38,13 @@ func New(options *Options) (*Runner, error) {
 	if len(options.Services) == 0 {
 		options.Services = append(options.Services, config.GetServiceNames()...)
 	}
+	if len(options.ExcludeServices) == 0 {
+		options.ExcludeServices = append(options.ExcludeServices, config.GetExcludeServiceNames()...)
+	}
 
 	// assign default services if not provided
 	if len(options.Services) == 0 {
-		options.Services = append(options.Services, defaultServies...)
+		options.Services = append(options.Services, defaultServices...)
 	}
 	if len(options.Providers) == 0 {
 		options.Providers = append(options.Providers, defaultProviders...)
@@ -57,6 +60,10 @@ func (r *Runner) Enumerate() {
 	if r.options.Services != nil {
 		services = r.options.Services
 	}
+	excludeServices := []string{}
+	if r.options.ExcludeServices != nil {
+		excludeServices = r.options.ExcludeServices
+	}
 
 	for _, item := range r.config {
 		if item == nil {
@@ -68,6 +75,9 @@ func (r *Runner) Enumerate() {
 		if len(services) > 0 {
 			item["services"] = strings.Join(services, ",")
 		}
+		if len(excludeServices) > 0 {
+			item["exclude_services"] = strings.Join(excludeServices, ",")
+		}
 		if r.options.ExtendedMetadata {
 			item["extended_metadata"] = "true"
 		}

pkg/providers/alibaba/alibaba.go

@@ -2,7 +2,6 @@ package alibaba
 
 import (
 	"context"
-	"strings"
 
 	"github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"
 	"github.com/projectdiscovery/cloudlist/pkg/schema"
@@ -42,23 +41,7 @@ func New(options schema.OptionBlock) (*Provider, error) {
 	id, _ := options.GetMetadata("id")
 	provider := &Provider{id: id}
 
-	supportedServicesMap := make(map[string]struct{})
-	for _, s := range Services {
-		supportedServicesMap[s] = struct{}{}
-	}
-	services := make(schema.ServiceMap)
-	if ss, ok := options.GetMetadata("services"); ok {
-		for _, s := range strings.Split(ss, ",") {
-			if _, ok := supportedServicesMap[s]; ok {
-				services[s] = struct{}{}
-			}
-		}
-	}
-	if len(services) == 0 {
-		for _, s := range Services {
-			services[s] = struct{}{}
-		}
-	}
+	services := options.ResolveServices(Services)
 	provider.services = services
 
 	if services.Has("instance") {

pkg/providers/arvancloud/arvancloud.go

@@ -2,7 +2,6 @@ package arvancloud
 
 import (
 	"context"
-	"strings"
 
 	r1c "git.arvancloud.ir/arvancloud/cdn-go-sdk"
 	"github.com/projectdiscovery/cloudlist/pkg/schema"
@@ -35,24 +34,7 @@ func New(options schema.OptionBlock) (*Provider, error) {
 	// Construct a new API object
 	api := r1c.NewAPIClient(configuration)
 
-	supportedServicesMap := make(map[string]struct{})
-	for _, s := range Services {
-		supportedServicesMap[s] = struct{}{}
-	}
-
-	services := make(schema.ServiceMap)
-	if ss, ok := options.GetMetadata("services"); ok {
-		for _, s := range strings.Split(ss, ",") {
-			if _, ok := supportedServicesMap[s]; ok {
-				services[s] = struct{}{}
-			}
-		}
-	}
-	if len(services) == 0 {
-		for _, s := range Services {
-			services[s] = struct{}{}
-		}
-	}
+	services := options.ResolveServices(Services)
 
 	return &Provider{id: id, client: api, services: services}, nil
 }

pkg/providers/aws/aws.go

@@ -83,25 +83,7 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error {
 		p.OrgDiscoveryRoleArn = orgRoleArn
 	}
 
-	supportedServicesMap := make(map[string]struct{})
-	for _, s := range Services {
-		supportedServicesMap[s] = struct{}{}
-	}
-	services := make(schema.ServiceMap)
-	if ss, ok := block.GetMetadata("services"); ok {
-		for _, s := range strings.Split(ss, ",") {
-			if _, ok := supportedServicesMap[s]; ok {
-				services[s] = struct{}{}
-			}
-		}
-	}
-	// if no services provided from -service flag, includes all services
-	if len(services) == 0 {
-		for _, s := range Services {
-			services[s] = struct{}{}
-		}
-	}
-	p.Services = services
+	p.Services = block.ResolveServices(Services)
 
 	if extendedMetadata, ok := block.GetMetadata("extended_metadata"); ok {
 		p.ExtendedMetadata = extendedMetadata == "true"

pkg/providers/aws/aws_test.go

@@ -7,6 +7,54 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+// TestParseOptionBlockExcludeServices covers exclude_services resolution:
+// the excluded entries are dropped from the effective set, exclusion composes
+// with the services allowlist, and unknown values are ignored.
+func TestParseOptionBlockExcludeServices(t *testing.T) {
+	tests := []struct {
+		name        string
+		block       schema.OptionBlock
+		wantPresent []string
+		wantAbsent  []string
+	}{
+		{
+			name:        "exclude from default-all set",
+			block:       schema.OptionBlock{"exclude_services": "s3,route53"},
+			wantPresent: []string{"ec2", "lambda"},
+			wantAbsent:  []string{"s3", "route53"},
+		},
+		{
+			name:        "exclude composes with services allowlist",
+			block:       schema.OptionBlock{"services": "ec2,s3,lambda", "exclude_services": "s3"},
+			wantPresent: []string{"ec2", "lambda"},
+			wantAbsent:  []string{"s3"},
+		},
+		{
+			name:        "whitespace is trimmed in both lists",
+			block:       schema.OptionBlock{"services": "ec2, s3, lambda", "exclude_services": "s3, lambda"},
+			wantPresent: []string{"ec2"},
+			wantAbsent:  []string{"s3", "lambda"},
+		},
+		{
+			name:        "unknown exclude value is ignored",
+			block:       schema.OptionBlock{"exclude_services": "not-a-service"},
+			wantPresent: []string{"ec2", "s3"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var opts ProviderOptions
+			require.NoError(t, opts.ParseOptionBlock(tt.block))
+			for _, s := range tt.wantPresent {
+				require.True(t, opts.Services.Has(s), "expected service %q to be present", s)
+			}
+			for _, s := range tt.wantAbsent {
+				require.False(t, opts.Services.Has(s), "expected service %q to be excluded", s)
+			}
+		})
+	}
+}
+
 // TestParseOptionBlock covers the static-credential pair validation that backs
 // keyless authentication: both keys present is valid, both omitted is valid
 // (keyless), and a half-configured pair is rejected.

pkg/providers/azure/azure.go

@@ -3,7 +3,6 @@ package azure
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions"
@@ -44,23 +43,7 @@ func New(options schema.OptionBlock) (*Provider, error) {
 	}
 
 	// Parse services
-	supportedServicesMap := make(map[string]struct{})
-	for _, s := range Services {
-		supportedServicesMap[s] = struct{}{}
-	}
-	services := make(schema.ServiceMap)
-	if ss, ok := options.GetMetadata("services"); ok {
-		for _, s := range strings.Split(ss, ",") {
-			if _, ok := supportedServicesMap[s]; ok {
-				services[s] = struct{}{}
-			}
-		}
-	}
-	if len(services) == 0 {
-		for _, s := range Services {
-			services[s] = struct{}{}
-		}
-	}
+	services := options.ResolveServices(Services)
 
 	provider := &Provider{
 		Credential: credential, // Track 2: use credential instead of authorizer

pkg/providers/cloudflare/cloudflare.go

@@ -3,7 +3,6 @@ package cloudflare
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/cloudflare/cloudflare-go"
 	"github.com/projectdiscovery/cloudlist/pkg/schema"
@@ -24,24 +23,7 @@ type Provider struct {
 func New(options schema.OptionBlock) (*Provider, error) {
 	id, _ := options.GetMetadata("id")
 
-	supportedServicesMap := make(map[string]struct{})
-	for _, s := range Services {
-		supportedServicesMap[s] = struct{}{}
-	}
-
-	services := make(schema.ServiceMap)
-	if ss, ok := options.GetMetadata("services"); ok {
-		for _, s := range strings.Split(ss, ",") {
-			if _, ok := supportedServicesMap[s]; ok {
-				services[s] = struct{}{}
-			}
-		}
-	}
-	if len(services) == 0 {
-		for _, s := range Services {
-			services[s] = struct{}{}
-		}
-	}
+	services := options.ResolveServices(Services)
 
 	// Parse extended metadata option
 	extendedMetadata := false

pkg/providers/custom/custom.go

@@ -43,17 +43,8 @@ func New(block schema.OptionBlock) (*Provider, error) {
 		return nil, err
 	}
 
-	supportedServicesMap := make(map[string]struct{})
-	for _, s := range Services {
-		supportedServicesMap[s] = struct{}{}
-	}
-
-	services := make(schema.ServiceMap)
-	for _, s := range Services {
-		services[s] = struct{}{}
-	}
 	client := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
-	return &Provider{client: client, id: options.Id, urlList: options.URLs, headerList: options.Headers, services: services}, nil
+	return &Provider{client: client, id: options.Id, urlList: options.URLs, headerList: options.Headers, services: options.Services}, nil
 }
 
 // Name returns the name of the provider
@@ -84,24 +75,7 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) {
 func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error {
 	p.Id, _ = block.GetMetadata("id")
 
-	supportedServicesMap := make(map[string]struct{})
-	for _, s := range Services {
-		supportedServicesMap[s] = struct{}{}
-	}
-	services := make(schema.ServiceMap)
-	if ss, ok := block.GetMetadata("services"); ok {
-		for _, s := range strings.Split(ss, ",") {
-			if _, ok := supportedServicesMap[s]; ok {
-				services[s] = struct{}{}
-			}
-		}
-	}
-	// if no services provided from -service flag, includes all services
-	if len(services) == 0 {
-		for _, s := range Services {
-			services[s] = struct{}{}
-		}
-	}
+	p.Services = block.ResolveServices(Services)
 
 	np, err := networkpolicy.New(networkpolicy.DefaultOptions)
 	if err != nil {