diff --git a/PROVIDERS.md b/PROVIDERS.md index d57ae3e..fa18154 100644 --- a/PROVIDERS.md +++ b/PROVIDERS.md @@ -1,5 +1,20 @@ # Providers +## Service filtering + +Every provider enumerates all of its supported services by default. Two optional, additive keys narrow that set (also available on the CLI as `-s`/`--service` and `-es`/`--exclude-service`): + +- `services` (list): allowlist. Only the listed services are enumerated. Unknown values are ignored. +- `exclude_services` (list): blocklist applied after `services` (or after the default-all set when `services` is omitted). Unknown values are ignored. + +```yaml +- provider: gcp + exclude_services: + - cloud-function + project_ids: + - my-project +``` + ### Amazon Web Services (AWS) Amazon Web Services can be integrated by using the following configuration block. diff --git a/README.md b/README.md index 2db8a3f..bc573f0 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,13 @@ CONFIGURATION: -pc, -provider-config string provider config file (default "$HOME/.config/cloudlist/provider-config.yaml") FILTERS: - -p, -provider value display results for given providers (comma-separated) (default linode,fastly,heroku,terraform,digitalocean,consul,cloudflare,hetzner,nomad,do,scw,openstack,alibaba,aws,gcp,namecheap,kubernetes,azure, custom) - -id string[] display results for given ids (comma-separated) - -host display only hostnames in results - -ip display only ips in results - -s, -service value query and display results from given service (comma-separated)) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway) - -ep, -exclude-private exclude private ips in cli output + -p, -provider value display results for given providers (comma-separated) (default linode,fastly,heroku,terraform,digitalocean,consul,cloudflare,hetzner,nomad,do,scw,openstack,alibaba,aws,gcp,namecheap,kubernetes,azure, custom) + -id string[] display results for given ids (comma-separated) + -host display only hostnames in results + -ip display only ips in results + -s, -service value query and display results from given service (comma-separated) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway) + -es, -exclude-service value services to skip for a provider (comma-separated) + -ep, -exclude-private exclude private ips in cli output UPDATE: -up, -update update cloudlist to latest version diff --git a/internal/runner/options.go b/internal/runner/options.go index b6dcff2..0c3bdc2 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,13 +25,14 @@ type Options struct { Version bool // Version returns the version of the tool. Verbose bool // Verbose prints verbose output. Hosts bool // Hosts specifies to fetch only DNS Names - IPAddress bool // IPAddress specifes to fetch only IP Addresses + IPAddress bool // IPAddress specifies to fetch only IP Addresses Config string // Config is the location of the config file. Output string // Output is the file to write found results too. ExcludePrivate bool // ExcludePrivate excludes private IPs from results Providers goflags.StringSlice // Providers specifies what providers to fetch assets for. Id goflags.StringSlice // Id specifies what id's to fetch assets for. Services goflags.StringSlice // Services specifies what services to fetch assets for a provider. + ExcludeServices goflags.StringSlice // ExcludeServices specifies what services to skip for a provider. ExtendedMetadata bool // ExtendedMetadata enables extended metadata for providers. ProviderConfig string // ProviderConfig is the location of the provider config file. DisableUpdateCheck bool // DisableUpdateCheck disable automatic update check @@ -40,7 +41,7 @@ type Options struct { var ( defaultConfigLocation = filepath.Join(userHomeDir(), ".config/cloudlist/config.yaml") defaultProviderConfigLocation = filepath.Join(userHomeDir(), ".config/cloudlist/provider-config.yaml") - defaultProviders, defaultServies = []string{}, []string{} + defaultProviders, defaultServices = []string{}, []string{} allowedProviders, allowedServices = []string{}, []string{} ) @@ -51,7 +52,7 @@ func init() { } for _, service := range inventory.GetServices() { - defaultServies = append(defaultServies, service) + defaultServices = append(defaultServices, service) allowedServices = append(allowedServices, service) } } @@ -84,7 +85,8 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Hosts, "host", false, "display only hostnames in results"), flagSet.BoolVar(&options.IPAddress, "ip", false, "display only ips in results"), flagSet.BoolVar(&options.ExtendedMetadata, "extended-metadata", false, "enable extended metadata for providers"), - flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated)) (default "+strings.Join(defaultServies, ",")+")", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated) (default "+strings.Join(defaultServices, ",")+")", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.ExcludeServices, "exclude-service", "es", nil, "services to skip for a provider (comma-separated)", goflags.CommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.ExcludePrivate, "exclude-private", "ep", false, "exclude private ips in cli output"), ) flagSet.CreateGroup("update", "Update", diff --git a/internal/runner/runner.go b/internal/runner/runner.go index a6b3986..35df918 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -38,10 +38,13 @@ func New(options *Options) (*Runner, error) { if len(options.Services) == 0 { options.Services = append(options.Services, config.GetServiceNames()...) } + if len(options.ExcludeServices) == 0 { + options.ExcludeServices = append(options.ExcludeServices, config.GetExcludeServiceNames()...) + } // assign default services if not provided if len(options.Services) == 0 { - options.Services = append(options.Services, defaultServies...) + options.Services = append(options.Services, defaultServices...) } if len(options.Providers) == 0 { options.Providers = append(options.Providers, defaultProviders...) @@ -57,6 +60,10 @@ func (r *Runner) Enumerate() { if r.options.Services != nil { services = r.options.Services } + excludeServices := []string{} + if r.options.ExcludeServices != nil { + excludeServices = r.options.ExcludeServices + } for _, item := range r.config { if item == nil { @@ -68,6 +75,9 @@ func (r *Runner) Enumerate() { if len(services) > 0 { item["services"] = strings.Join(services, ",") } + if len(excludeServices) > 0 { + item["exclude_services"] = strings.Join(excludeServices, ",") + } if r.options.ExtendedMetadata { item["extended_metadata"] = "true" } diff --git a/pkg/providers/alibaba/alibaba.go b/pkg/providers/alibaba/alibaba.go index f104241..9177e53 100644 --- a/pkg/providers/alibaba/alibaba.go +++ b/pkg/providers/alibaba/alibaba.go @@ -2,7 +2,6 @@ package alibaba import ( "context" - "strings" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -42,23 +41,7 @@ func New(options schema.OptionBlock) (*Provider, error) { id, _ := options.GetMetadata("id") provider := &Provider{id: id} - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) provider.services = services if services.Has("instance") { diff --git a/pkg/providers/arvancloud/arvancloud.go b/pkg/providers/arvancloud/arvancloud.go index 0b0aa07..e13f038 100644 --- a/pkg/providers/arvancloud/arvancloud.go +++ b/pkg/providers/arvancloud/arvancloud.go @@ -2,7 +2,6 @@ package arvancloud import ( "context" - "strings" r1c "git.arvancloud.ir/arvancloud/cdn-go-sdk" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -35,24 +34,7 @@ func New(options schema.OptionBlock) (*Provider, error) { // Construct a new API object api := r1c.NewAPIClient(configuration) - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: api, services: services}, nil } diff --git a/pkg/providers/aws/aws.go b/pkg/providers/aws/aws.go index 8d30630..61e1daf 100644 --- a/pkg/providers/aws/aws.go +++ b/pkg/providers/aws/aws.go @@ -83,25 +83,7 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error { p.OrgDiscoveryRoleArn = orgRoleArn } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := block.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - // if no services provided from -service flag, includes all services - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - p.Services = services + p.Services = block.ResolveServices(Services) if extendedMetadata, ok := block.GetMetadata("extended_metadata"); ok { p.ExtendedMetadata = extendedMetadata == "true" diff --git a/pkg/providers/aws/aws_test.go b/pkg/providers/aws/aws_test.go index 8d4afe1..e833696 100644 --- a/pkg/providers/aws/aws_test.go +++ b/pkg/providers/aws/aws_test.go @@ -7,6 +7,54 @@ import ( "github.com/stretchr/testify/require" ) +// TestParseOptionBlockExcludeServices covers exclude_services resolution: +// the excluded entries are dropped from the effective set, exclusion composes +// with the services allowlist, and unknown values are ignored. +func TestParseOptionBlockExcludeServices(t *testing.T) { + tests := []struct { + name string + block schema.OptionBlock + wantPresent []string + wantAbsent []string + }{ + { + name: "exclude from default-all set", + block: schema.OptionBlock{"exclude_services": "s3,route53"}, + wantPresent: []string{"ec2", "lambda"}, + wantAbsent: []string{"s3", "route53"}, + }, + { + name: "exclude composes with services allowlist", + block: schema.OptionBlock{"services": "ec2,s3,lambda", "exclude_services": "s3"}, + wantPresent: []string{"ec2", "lambda"}, + wantAbsent: []string{"s3"}, + }, + { + name: "whitespace is trimmed in both lists", + block: schema.OptionBlock{"services": "ec2, s3, lambda", "exclude_services": "s3, lambda"}, + wantPresent: []string{"ec2"}, + wantAbsent: []string{"s3", "lambda"}, + }, + { + name: "unknown exclude value is ignored", + block: schema.OptionBlock{"exclude_services": "not-a-service"}, + wantPresent: []string{"ec2", "s3"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var opts ProviderOptions + require.NoError(t, opts.ParseOptionBlock(tt.block)) + for _, s := range tt.wantPresent { + require.True(t, opts.Services.Has(s), "expected service %q to be present", s) + } + for _, s := range tt.wantAbsent { + require.False(t, opts.Services.Has(s), "expected service %q to be excluded", s) + } + }) + } +} + // TestParseOptionBlock covers the static-credential pair validation that backs // keyless authentication: both keys present is valid, both omitted is valid // (keyless), and a half-configured pair is rejected. diff --git a/pkg/providers/azure/azure.go b/pkg/providers/azure/azure.go index 3821732..9622122 100644 --- a/pkg/providers/azure/azure.go +++ b/pkg/providers/azure/azure.go @@ -3,7 +3,6 @@ package azure import ( "context" "fmt" - "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions" @@ -44,23 +43,7 @@ func New(options schema.OptionBlock) (*Provider, error) { } // Parse services - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) provider := &Provider{ Credential: credential, // Track 2: use credential instead of authorizer diff --git a/pkg/providers/cloudflare/cloudflare.go b/pkg/providers/cloudflare/cloudflare.go index aa2f7a1..4a70def 100644 --- a/pkg/providers/cloudflare/cloudflare.go +++ b/pkg/providers/cloudflare/cloudflare.go @@ -3,7 +3,6 @@ package cloudflare import ( "context" "fmt" - "strings" "github.com/cloudflare/cloudflare-go" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -24,24 +23,7 @@ type Provider struct { func New(options schema.OptionBlock) (*Provider, error) { id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) // Parse extended metadata option extendedMetadata := false diff --git a/pkg/providers/custom/custom.go b/pkg/providers/custom/custom.go index 11b2206..a29b8cc 100644 --- a/pkg/providers/custom/custom.go +++ b/pkg/providers/custom/custom.go @@ -43,17 +43,8 @@ func New(block schema.OptionBlock) (*Provider, error) { return nil, err } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - - services := make(schema.ServiceMap) - for _, s := range Services { - services[s] = struct{}{} - } client := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle) - return &Provider{client: client, id: options.Id, urlList: options.URLs, headerList: options.Headers, services: services}, nil + return &Provider{client: client, id: options.Id, urlList: options.URLs, headerList: options.Headers, services: options.Services}, nil } // Name returns the name of the provider @@ -84,24 +75,7 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error { p.Id, _ = block.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := block.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - // if no services provided from -service flag, includes all services - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + p.Services = block.ResolveServices(Services) np, err := networkpolicy.New(networkpolicy.DefaultOptions) if err != nil { diff --git a/pkg/providers/digitalocean/digitalocean.go b/pkg/providers/digitalocean/digitalocean.go index 364be84..6cbfd83 100644 --- a/pkg/providers/digitalocean/digitalocean.go +++ b/pkg/providers/digitalocean/digitalocean.go @@ -2,7 +2,6 @@ package digitalocean import ( "context" - "strings" "github.com/digitalocean/godo" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -26,23 +25,7 @@ func New(options schema.OptionBlock) (*Provider, error) { } id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) // Check for extended metadata option extendedMetadata := false diff --git a/pkg/providers/dnssimple/dnssimple.go b/pkg/providers/dnssimple/dnssimple.go index 285b183..e7bcc25 100644 --- a/pkg/providers/dnssimple/dnssimple.go +++ b/pkg/providers/dnssimple/dnssimple.go @@ -3,7 +3,6 @@ package dnssimple import ( "context" "fmt" - "strings" "github.com/dnsimple/dnsimple-go/dnsimple" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -53,23 +52,7 @@ func New(options schema.OptionBlock) (*Provider, error) { client := dnsimple.NewClient(dnsimple.StaticTokenHTTPClient(context.Background(), token)) // Configure services - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) provider := &Provider{ id: id, diff --git a/pkg/providers/gcp/gcp.go b/pkg/providers/gcp/gcp.go index e44588c..5f26e55 100644 --- a/pkg/providers/gcp/gcp.go +++ b/pkg/providers/gcp/gcp.go @@ -239,23 +239,7 @@ func newIndividualProvider(options schema.OptionBlock, id, JSONData string) (*Pr provider.extendedMetadata = extendedMetadata == "true" } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) provider.services = services configuredProjects := getProjectIDsFromOptions(options) @@ -413,7 +397,7 @@ func (p *OrganizationProvider) Resources(ctx context.Context) (*schema.Resources var projectResources *schema.Resources var err error - // if projects has all, then get all assets + // if projects has all, then get all assets if p.services.Has("all") { projectResources, err = p.getAllAssets(ctx, parent) if err != nil { @@ -665,6 +649,7 @@ func newOrganizationProvider(options schema.OptionBlock, id, JSONData, organizat services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } @@ -674,6 +659,22 @@ func newOrganizationProvider(options schema.OptionBlock, id, JSONData, organizat // Default to all services for organization-level discovery services["all"] = struct{}{} } + if es, ok := options.GetMetadata("exclude_services"); ok { + // The "all" sentinel routes to the comprehensive Asset API path, which + // cannot drop individual services; expand it to the concrete set first + // so exclusions take effect. + if services.Has("all") { + delete(services, "all") + for _, s := range allServices { + if s != "all" { + services[s] = struct{}{} + } + } + } + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } provider.services = services // Extract short-lived credentials configuration diff --git a/pkg/providers/heroku/heroku.go b/pkg/providers/heroku/heroku.go index ac2a59d..c55564e 100644 --- a/pkg/providers/heroku/heroku.go +++ b/pkg/providers/heroku/heroku.go @@ -2,7 +2,6 @@ package heroku import ( "context" - "strings" heroku "github.com/heroku/heroku-go/v5" @@ -33,23 +32,7 @@ func New(options schema.OptionBlock) (*Provider, error) { heroku.DefaultTransport.BearerToken = token - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: heroku.NewService(heroku.DefaultClient), services: services}, nil } diff --git a/pkg/providers/hetzner/hetzner.go b/pkg/providers/hetzner/hetzner.go index 25ff5f0..f7bd07f 100644 --- a/pkg/providers/hetzner/hetzner.go +++ b/pkg/providers/hetzner/hetzner.go @@ -2,7 +2,6 @@ package hetzner import ( "context" - "strings" hetzner "github.com/hetznercloud/hcloud-go/hcloud" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -32,23 +31,7 @@ func New(options schema.OptionBlock) (*Provider, error) { id, _ := options.GetMetadata("id") opts := hetzner.WithToken(token) - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: hetzner.NewClient(opts), services: services}, nil } diff --git a/pkg/providers/k8s/kubernetes.go b/pkg/providers/k8s/kubernetes.go index 659fd7d..e5ba2fe 100644 --- a/pkg/providers/k8s/kubernetes.go +++ b/pkg/providers/k8s/kubernetes.go @@ -4,7 +4,6 @@ import ( "context" "encoding/base64" "fmt" - "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" "github.com/projectdiscovery/utils/errkit" @@ -64,23 +63,7 @@ func New(options schema.OptionBlock) (*Provider, error) { return nil, errkit.Wrap(err, "could not create kubernetes clientset") } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) var providerExtendedMetadata bool if extendedMetadata, ok := options.GetMetadata("extended_metadata"); ok { providerExtendedMetadata = extendedMetadata == "true" diff --git a/pkg/providers/linode/linode.go b/pkg/providers/linode/linode.go index 389fa11..b675b6d 100644 --- a/pkg/providers/linode/linode.go +++ b/pkg/providers/linode/linode.go @@ -3,7 +3,6 @@ package linode import ( "context" "net/http" - "strings" "github.com/linode/linodego" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -42,23 +41,7 @@ func New(options schema.OptionBlock) (*Provider, error) { client := linodego.NewClient(oc) - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: &client, services: services}, nil } diff --git a/pkg/providers/namecheap/namecheap.go b/pkg/providers/namecheap/namecheap.go index d1d86b8..96f51df 100644 --- a/pkg/providers/namecheap/namecheap.go +++ b/pkg/providers/namecheap/namecheap.go @@ -2,7 +2,6 @@ package namecheap import ( "context" - "strings" "github.com/namecheap/go-namecheap-sdk/v2/namecheap" @@ -52,23 +51,7 @@ func New(options schema.OptionBlock) (*Provider, error) { UseSandbox: false, } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: namecheap.NewClient(&clientOptions), services: services}, nil } diff --git a/pkg/providers/openstack/openstack.go b/pkg/providers/openstack/openstack.go index f836d53..eecb071 100644 --- a/pkg/providers/openstack/openstack.go +++ b/pkg/providers/openstack/openstack.go @@ -2,7 +2,6 @@ package openstack import ( "context" - "strings" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack" @@ -82,23 +81,7 @@ func New(options schema.OptionBlock) (*Provider, error) { return nil, err } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: client, services: services}, nil } diff --git a/pkg/providers/ovh/ovh.go b/pkg/providers/ovh/ovh.go index 1fcb3da..ece059c 100644 --- a/pkg/providers/ovh/ovh.go +++ b/pkg/providers/ovh/ovh.go @@ -3,7 +3,6 @@ package ovh import ( "context" "net/http" - "strings" "time" "github.com/ovh/go-ovh/ovh" @@ -24,21 +23,7 @@ func New(options schema.OptionBlock) (*Provider, error) { id, _ := options.GetMetadata("id") // service selection - supported := map[string]struct{}{"dns": {}} - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supported[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) // OVH endpoint (default ovh-eu) endpoint := "ovh-eu" diff --git a/pkg/providers/scaleway/scaleway.go b/pkg/providers/scaleway/scaleway.go index 6277b52..4c9bc04 100644 --- a/pkg/providers/scaleway/scaleway.go +++ b/pkg/providers/scaleway/scaleway.go @@ -2,7 +2,6 @@ package scaleway import ( "context" - "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" @@ -29,23 +28,7 @@ func New(options schema.OptionBlock) (*Provider, error) { return nil, &schema.ErrNoSuchKey{Name: apiAccessToken} } id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) client, err := scw.NewClient(scw.WithAuth(accessKey, accessToken)) if err != nil { diff --git a/pkg/providers/terraform/terraform.go b/pkg/providers/terraform/terraform.go index 9adcb09..80ec50a 100644 --- a/pkg/providers/terraform/terraform.go +++ b/pkg/providers/terraform/terraform.go @@ -2,7 +2,6 @@ package terraform import ( "context" - "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" ) @@ -29,23 +28,7 @@ func New(options schema.OptionBlock) (*Provider, error) { } id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) return &Provider{path: StatePathFile, id: id, services: services}, nil } diff --git a/pkg/providers/vercel/vercel.go b/pkg/providers/vercel/vercel.go index 48aecd4..1427dc6 100644 --- a/pkg/providers/vercel/vercel.go +++ b/pkg/providers/vercel/vercel.go @@ -2,7 +2,6 @@ package vercel import ( "context" - "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" ) @@ -25,23 +24,7 @@ func New(options schema.OptionBlock) (*Provider, error) { teamID, _ := options.GetMetadata(apiTeamID) id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } + services := options.ResolveServices(Services) client := newAPIClient(newClientConfig{ Token: accessKey, diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 67ad620..38ec6ad 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -212,18 +212,30 @@ type Options []OptionBlock // GetServiceNames returns the services from the options func (o Options) GetServiceNames() []string { - services := make([]string, 0) + return o.collectListValues("services") +} + +// GetExcludeServiceNames returns the excluded services from the options +func (o Options) GetExcludeServiceNames() []string { + return o.collectListValues("exclude_services") +} + +// collectListValues flattens a comma-separated option key across all blocks, +// trimming blanks. +func (o Options) collectListValues(key string) []string { + values := make([]string, 0) for _, option := range o { - if serviceNameList, ok := option["services"]; ok { - for _, serviceName := range strings.Split(serviceNameList, ",") { - trimmedServiceName := strings.TrimSpace(serviceName) - if trimmedServiceName != "" { - services = append(services, trimmedServiceName) - } + list, ok := option[key] + if !ok { + continue + } + for _, value := range strings.Split(list, ",") { + if trimmed := strings.TrimSpace(value); trimmed != "" { + values = append(values, trimmed) } } } - return services + return values } // OptionBlock is a single option on which operation is possible @@ -240,7 +252,7 @@ func (ob *OptionBlock) UnmarshalYAML(unmarshal func(interface{}) error) error { // Convert raw map to OptionBlock and handle special cases for key, value := range rawMap { switch key { - case "account_ids", "exclude_account_ids", "urls", "services", "project_ids", "exclude_project_ids": + case "account_ids", "exclude_account_ids", "urls", "services", "exclude_services", "project_ids", "exclude_project_ids": if valueArr, ok := value.([]interface{}); ok { var strArr []string for _, v := range valueArr { @@ -296,6 +308,43 @@ func (o OptionBlock) GetMetadata(key string) (string, bool) { return data, true } +// ResolveServices computes the effective service set for a provider from the +// option block. It starts from the `services` allowlist (falling back to all +// supported services when the allowlist is empty), then removes any +// `exclude_services` entries. Entries are whitespace-trimmed and values that +// are not in supported are ignored. +func (o OptionBlock) ResolveServices(supported []string) ServiceMap { + supportedSet := make(map[string]struct{}, len(supported)) + for _, s := range supported { + supportedSet[s] = struct{}{} + } + + services := make(ServiceMap) + if allow, ok := o.GetMetadata("services"); ok { + for _, s := range strings.Split(allow, ",") { + s = strings.TrimSpace(s) + if _, ok := supportedSet[s]; ok { + services[s] = struct{}{} + } + } + } + + // default to all supported services when no allowlist is provided + if len(services) == 0 { + for _, s := range supported { + services[s] = struct{}{} + } + } + + if exclude, ok := o.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(exclude, ",") { + delete(services, strings.TrimSpace(s)) + } + } + + return services +} + type ServiceMap map[string]struct{} func (s ServiceMap) Has(service string) bool { diff --git a/pkg/schema/schema_test.go b/pkg/schema/schema_test.go index 86d91c5..ebb16f7 100644 --- a/pkg/schema/schema_test.go +++ b/pkg/schema/schema_test.go @@ -75,6 +75,92 @@ func TestOptionBlockZeroPadsExcludeAccountIDs(t *testing.T) { require.Equal(t, "012345678901", value) } +func TestOptionBlockParsesExcludeServices(t *testing.T) { + data := ` +- provider: gcp + exclude_services: + - cloud-function + - gke +` + var options Options + err := yaml.Unmarshal([]byte(data), &options) + require.NoError(t, err) + require.Len(t, options, 1) + + value, ok := options[0].GetMetadata("exclude_services") + require.True(t, ok) + require.Equal(t, "cloud-function,gke", value) + require.Equal(t, []string{"cloud-function", "gke"}, options.GetExcludeServiceNames()) +} + +func TestResolveServices(t *testing.T) { + supported := []string{"ec2", "s3", "lambda", "route53"} + + tests := []struct { + name string + block OptionBlock + wantPresent []string + wantAbsent []string + }{ + { + name: "no options defaults to all supported", + block: OptionBlock{}, + wantPresent: supported, + }, + { + name: "allowlist keeps only listed services", + block: OptionBlock{"services": "ec2,s3"}, + wantPresent: []string{"ec2", "s3"}, + wantAbsent: []string{"lambda", "route53"}, + }, + { + name: "exclude drops from default-all set", + block: OptionBlock{"exclude_services": "s3,route53"}, + wantPresent: []string{"ec2", "lambda"}, + wantAbsent: []string{"s3", "route53"}, + }, + { + name: "exclude composes with allowlist", + block: OptionBlock{"services": "ec2,s3,lambda", "exclude_services": "s3"}, + wantPresent: []string{"ec2", "lambda"}, + wantAbsent: []string{"s3"}, + }, + { + name: "whitespace is trimmed in both lists", + block: OptionBlock{"services": "ec2, s3, lambda", "exclude_services": "s3, lambda"}, + wantPresent: []string{"ec2"}, + wantAbsent: []string{"s3", "lambda"}, + }, + { + name: "unknown allowlist values are ignored and fall back to all", + block: OptionBlock{"services": "not-a-service"}, + wantPresent: supported, + }, + { + name: "unknown exclude values are ignored", + block: OptionBlock{"exclude_services": "not-a-service"}, + wantPresent: supported, + }, + { + name: "excluding every service yields an empty set", + block: OptionBlock{"exclude_services": "ec2,s3,lambda,route53"}, + wantAbsent: supported, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + services := tt.block.ResolveServices(supported) + for _, s := range tt.wantPresent { + require.Truef(t, services.Has(s), "expected service %q to be present", s) + } + for _, s := range tt.wantAbsent { + require.Falsef(t, services.Has(s), "expected service %q to be absent", s) + } + }) + } +} + func TestOptionBlockScalarFallback(t *testing.T) { data := ` - provider: aws