From 4523fb237286d8c463e46379d6a7b7824c037c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 24 Jun 2026 15:35:15 +0300 Subject: [PATCH 1/4] feat: add exclude_services filter (closes #749) --- PROVIDERS.md | 15 ++++++++ README.md | 13 +++---- internal/runner/options.go | 10 +++--- internal/runner/runner.go | 12 ++++++- pkg/providers/alibaba/alibaba.go | 5 +++ pkg/providers/arvancloud/arvancloud.go | 5 +++ pkg/providers/aws/aws.go | 5 +++ pkg/providers/aws/aws_test.go | 42 ++++++++++++++++++++++ pkg/providers/azure/azure.go | 5 +++ pkg/providers/cloudflare/cloudflare.go | 5 +++ pkg/providers/custom/custom.go | 5 +++ pkg/providers/digitalocean/digitalocean.go | 5 +++ pkg/providers/dnssimple/dnssimple.go | 5 +++ pkg/providers/gcp/gcp.go | 23 +++++++++++- pkg/providers/heroku/heroku.go | 5 +++ pkg/providers/hetzner/hetzner.go | 5 +++ pkg/providers/k8s/kubernetes.go | 5 +++ pkg/providers/linode/linode.go | 5 +++ pkg/providers/namecheap/namecheap.go | 5 +++ pkg/providers/openstack/openstack.go | 5 +++ pkg/providers/ovh/ovh.go | 5 +++ pkg/providers/scaleway/scaleway.go | 5 +++ pkg/providers/terraform/terraform.go | 5 +++ pkg/providers/vercel/vercel.go | 5 +++ pkg/schema/schema.go | 30 +++++++++++----- pkg/schema/schema_test.go | 18 ++++++++++ 26 files changed, 232 insertions(+), 21 deletions(-) diff --git a/PROVIDERS.md b/PROVIDERS.md index d57ae3e..b70e215 100644 --- a/PROVIDERS.md +++ b/PROVIDERS.md @@ -1,5 +1,20 @@ # Providers +### Service filtering + +Every provider enumerates all of its supported services by default. Two optional, additive keys narrow that set (also available on the CLI as `-s`/`--service` and `-es`/`--exclude-service`): + +- `services` (list): allowlist. Only the listed services are enumerated. Unknown values are ignored. +- `exclude_services` (list): blocklist applied after `services` (or after the default-all set when `services` is omitted). Unknown values are ignored. + +```yaml +- provider: gcp + exclude_services: + - cloud-function + project_ids: + - my-project +``` + ### Amazon Web Services (AWS) Amazon Web Services can be integrated by using the following configuration block. diff --git a/README.md b/README.md index 2db8a3f..f5f3840 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,13 @@ CONFIGURATION: -pc, -provider-config string provider config file (default "$HOME/.config/cloudlist/provider-config.yaml") FILTERS: - -p, -provider value display results for given providers (comma-separated) (default linode,fastly,heroku,terraform,digitalocean,consul,cloudflare,hetzner,nomad,do,scw,openstack,alibaba,aws,gcp,namecheap,kubernetes,azure, custom) - -id string[] display results for given ids (comma-separated) - -host display only hostnames in results - -ip display only ips in results - -s, -service value query and display results from given service (comma-separated)) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway) - -ep, -exclude-private exclude private ips in cli output + -p, -provider value display results for given providers (comma-separated) (default linode,fastly,heroku,terraform,digitalocean,consul,cloudflare,hetzner,nomad,do,scw,openstack,alibaba,aws,gcp,namecheap,kubernetes,azure, custom) + -id string[] display results for given ids (comma-separated) + -host display only hostnames in results + -ip display only ips in results + -s, -service value query and display results from given service (comma-separated)) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway) + -es, -exclude-service value services to skip for a provider (comma-separated) + -ep, -exclude-private exclude private ips in cli output UPDATE: -up, -update update cloudlist to latest version diff --git a/internal/runner/options.go b/internal/runner/options.go index b6dcff2..028d895 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,13 +25,14 @@ type Options struct { Version bool // Version returns the version of the tool. Verbose bool // Verbose prints verbose output. Hosts bool // Hosts specifies to fetch only DNS Names - IPAddress bool // IPAddress specifes to fetch only IP Addresses + IPAddress bool // IPAddress specifies to fetch only IP Addresses Config string // Config is the location of the config file. Output string // Output is the file to write found results too. ExcludePrivate bool // ExcludePrivate excludes private IPs from results Providers goflags.StringSlice // Providers specifies what providers to fetch assets for. Id goflags.StringSlice // Id specifies what id's to fetch assets for. Services goflags.StringSlice // Services specifies what services to fetch assets for a provider. + ExcludeServices goflags.StringSlice // ExcludeServices specifies what services to skip for a provider. ExtendedMetadata bool // ExtendedMetadata enables extended metadata for providers. ProviderConfig string // ProviderConfig is the location of the provider config file. DisableUpdateCheck bool // DisableUpdateCheck disable automatic update check @@ -40,7 +41,7 @@ type Options struct { var ( defaultConfigLocation = filepath.Join(userHomeDir(), ".config/cloudlist/config.yaml") defaultProviderConfigLocation = filepath.Join(userHomeDir(), ".config/cloudlist/provider-config.yaml") - defaultProviders, defaultServies = []string{}, []string{} + defaultProviders, defaultServices = []string{}, []string{} allowedProviders, allowedServices = []string{}, []string{} ) @@ -51,7 +52,7 @@ func init() { } for _, service := range inventory.GetServices() { - defaultServies = append(defaultServies, service) + defaultServices = append(defaultServices, service) allowedServices = append(allowedServices, service) } } @@ -84,7 +85,8 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Hosts, "host", false, "display only hostnames in results"), flagSet.BoolVar(&options.IPAddress, "ip", false, "display only ips in results"), flagSet.BoolVar(&options.ExtendedMetadata, "extended-metadata", false, "enable extended metadata for providers"), - flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated)) (default "+strings.Join(defaultServies, ",")+")", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated)) (default "+strings.Join(defaultServices, ",")+")", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.ExcludeServices, "exclude-service", "es", nil, "services to skip for a provider (comma-separated)", goflags.CommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.ExcludePrivate, "exclude-private", "ep", false, "exclude private ips in cli output"), ) flagSet.CreateGroup("update", "Update", diff --git a/internal/runner/runner.go b/internal/runner/runner.go index a6b3986..35df918 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -38,10 +38,13 @@ func New(options *Options) (*Runner, error) { if len(options.Services) == 0 { options.Services = append(options.Services, config.GetServiceNames()...) } + if len(options.ExcludeServices) == 0 { + options.ExcludeServices = append(options.ExcludeServices, config.GetExcludeServiceNames()...) + } // assign default services if not provided if len(options.Services) == 0 { - options.Services = append(options.Services, defaultServies...) + options.Services = append(options.Services, defaultServices...) } if len(options.Providers) == 0 { options.Providers = append(options.Providers, defaultProviders...) @@ -57,6 +60,10 @@ func (r *Runner) Enumerate() { if r.options.Services != nil { services = r.options.Services } + excludeServices := []string{} + if r.options.ExcludeServices != nil { + excludeServices = r.options.ExcludeServices + } for _, item := range r.config { if item == nil { @@ -68,6 +75,9 @@ func (r *Runner) Enumerate() { if len(services) > 0 { item["services"] = strings.Join(services, ",") } + if len(excludeServices) > 0 { + item["exclude_services"] = strings.Join(excludeServices, ",") + } if r.options.ExtendedMetadata { item["extended_metadata"] = "true" } diff --git a/pkg/providers/alibaba/alibaba.go b/pkg/providers/alibaba/alibaba.go index f104241..e346665 100644 --- a/pkg/providers/alibaba/alibaba.go +++ b/pkg/providers/alibaba/alibaba.go @@ -59,6 +59,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } provider.services = services if services.Has("instance") { diff --git a/pkg/providers/arvancloud/arvancloud.go b/pkg/providers/arvancloud/arvancloud.go index 0b0aa07..5338d05 100644 --- a/pkg/providers/arvancloud/arvancloud.go +++ b/pkg/providers/arvancloud/arvancloud.go @@ -53,6 +53,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } return &Provider{id: id, client: api, services: services}, nil } diff --git a/pkg/providers/aws/aws.go b/pkg/providers/aws/aws.go index 8d30630..ac0ef9b 100644 --- a/pkg/providers/aws/aws.go +++ b/pkg/providers/aws/aws.go @@ -101,6 +101,11 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error { services[s] = struct{}{} } } + if es, ok := block.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } p.Services = services if extendedMetadata, ok := block.GetMetadata("extended_metadata"); ok { diff --git a/pkg/providers/aws/aws_test.go b/pkg/providers/aws/aws_test.go index 8d4afe1..78ceac6 100644 --- a/pkg/providers/aws/aws_test.go +++ b/pkg/providers/aws/aws_test.go @@ -7,6 +7,48 @@ import ( "github.com/stretchr/testify/require" ) +// TestParseOptionBlockExcludeServices covers exclude_services resolution: +// the excluded entries are dropped from the effective set, exclusion composes +// with the services allowlist, and unknown values are ignored. +func TestParseOptionBlockExcludeServices(t *testing.T) { + tests := []struct { + name string + block schema.OptionBlock + wantPresent []string + wantAbsent []string + }{ + { + name: "exclude from default-all set", + block: schema.OptionBlock{"exclude_services": "s3,route53"}, + wantPresent: []string{"ec2", "lambda"}, + wantAbsent: []string{"s3", "route53"}, + }, + { + name: "exclude composes with services allowlist", + block: schema.OptionBlock{"services": "ec2,s3,lambda", "exclude_services": "s3"}, + wantPresent: []string{"ec2", "lambda"}, + wantAbsent: []string{"s3", "route53"}, + }, + { + name: "unknown exclude value is ignored", + block: schema.OptionBlock{"exclude_services": "not-a-service"}, + wantPresent: []string{"ec2", "s3"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var opts ProviderOptions + require.NoError(t, opts.ParseOptionBlock(tt.block)) + for _, s := range tt.wantPresent { + require.True(t, opts.Services.Has(s), "expected service %q to be present", s) + } + for _, s := range tt.wantAbsent { + require.False(t, opts.Services.Has(s), "expected service %q to be excluded", s) + } + }) + } +} + // TestParseOptionBlock covers the static-credential pair validation that backs // keyless authentication: both keys present is valid, both omitted is valid // (keyless), and a half-configured pair is rejected. diff --git a/pkg/providers/azure/azure.go b/pkg/providers/azure/azure.go index 3821732..d72bd96 100644 --- a/pkg/providers/azure/azure.go +++ b/pkg/providers/azure/azure.go @@ -61,6 +61,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } provider := &Provider{ Credential: credential, // Track 2: use credential instead of authorizer diff --git a/pkg/providers/cloudflare/cloudflare.go b/pkg/providers/cloudflare/cloudflare.go index aa2f7a1..65c79fb 100644 --- a/pkg/providers/cloudflare/cloudflare.go +++ b/pkg/providers/cloudflare/cloudflare.go @@ -42,6 +42,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } // Parse extended metadata option extendedMetadata := false diff --git a/pkg/providers/custom/custom.go b/pkg/providers/custom/custom.go index 11b2206..2376a87 100644 --- a/pkg/providers/custom/custom.go +++ b/pkg/providers/custom/custom.go @@ -102,6 +102,11 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error { services[s] = struct{}{} } } + if es, ok := block.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } np, err := networkpolicy.New(networkpolicy.DefaultOptions) if err != nil { diff --git a/pkg/providers/digitalocean/digitalocean.go b/pkg/providers/digitalocean/digitalocean.go index 364be84..e3dd965 100644 --- a/pkg/providers/digitalocean/digitalocean.go +++ b/pkg/providers/digitalocean/digitalocean.go @@ -43,6 +43,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } // Check for extended metadata option extendedMetadata := false diff --git a/pkg/providers/dnssimple/dnssimple.go b/pkg/providers/dnssimple/dnssimple.go index 285b183..fb4acb7 100644 --- a/pkg/providers/dnssimple/dnssimple.go +++ b/pkg/providers/dnssimple/dnssimple.go @@ -70,6 +70,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } provider := &Provider{ id: id, diff --git a/pkg/providers/gcp/gcp.go b/pkg/providers/gcp/gcp.go index e44588c..b7469ff 100644 --- a/pkg/providers/gcp/gcp.go +++ b/pkg/providers/gcp/gcp.go @@ -256,6 +256,11 @@ func newIndividualProvider(options schema.OptionBlock, id, JSONData string) (*Pr services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } provider.services = services configuredProjects := getProjectIDsFromOptions(options) @@ -413,7 +418,7 @@ func (p *OrganizationProvider) Resources(ctx context.Context) (*schema.Resources var projectResources *schema.Resources var err error - // if projects has all, then get all assets + // if projects has all, then get all assets if p.services.Has("all") { projectResources, err = p.getAllAssets(ctx, parent) if err != nil { @@ -674,6 +679,22 @@ func newOrganizationProvider(options schema.OptionBlock, id, JSONData, organizat // Default to all services for organization-level discovery services["all"] = struct{}{} } + if es, ok := options.GetMetadata("exclude_services"); ok { + // The "all" sentinel routes to the comprehensive Asset API path, which + // cannot drop individual services; expand it to the concrete set first + // so exclusions take effect. + if services.Has("all") { + delete(services, "all") + for _, s := range allServices { + if s != "all" { + services[s] = struct{}{} + } + } + } + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } provider.services = services // Extract short-lived credentials configuration diff --git a/pkg/providers/heroku/heroku.go b/pkg/providers/heroku/heroku.go index ac2a59d..6a5b3f8 100644 --- a/pkg/providers/heroku/heroku.go +++ b/pkg/providers/heroku/heroku.go @@ -50,6 +50,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } return &Provider{id: id, client: heroku.NewService(heroku.DefaultClient), services: services}, nil } diff --git a/pkg/providers/hetzner/hetzner.go b/pkg/providers/hetzner/hetzner.go index 25ff5f0..28d18fb 100644 --- a/pkg/providers/hetzner/hetzner.go +++ b/pkg/providers/hetzner/hetzner.go @@ -49,6 +49,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } return &Provider{id: id, client: hetzner.NewClient(opts), services: services}, nil } diff --git a/pkg/providers/k8s/kubernetes.go b/pkg/providers/k8s/kubernetes.go index 659fd7d..2a5ee01 100644 --- a/pkg/providers/k8s/kubernetes.go +++ b/pkg/providers/k8s/kubernetes.go @@ -81,6 +81,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } var providerExtendedMetadata bool if extendedMetadata, ok := options.GetMetadata("extended_metadata"); ok { providerExtendedMetadata = extendedMetadata == "true" diff --git a/pkg/providers/linode/linode.go b/pkg/providers/linode/linode.go index 389fa11..4d3d401 100644 --- a/pkg/providers/linode/linode.go +++ b/pkg/providers/linode/linode.go @@ -59,6 +59,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } return &Provider{id: id, client: &client, services: services}, nil } diff --git a/pkg/providers/namecheap/namecheap.go b/pkg/providers/namecheap/namecheap.go index d1d86b8..26d2fe8 100644 --- a/pkg/providers/namecheap/namecheap.go +++ b/pkg/providers/namecheap/namecheap.go @@ -69,6 +69,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } return &Provider{id: id, client: namecheap.NewClient(&clientOptions), services: services}, nil } diff --git a/pkg/providers/openstack/openstack.go b/pkg/providers/openstack/openstack.go index f836d53..b392b10 100644 --- a/pkg/providers/openstack/openstack.go +++ b/pkg/providers/openstack/openstack.go @@ -99,6 +99,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } return &Provider{id: id, client: client, services: services}, nil } diff --git a/pkg/providers/ovh/ovh.go b/pkg/providers/ovh/ovh.go index 1fcb3da..9eda436 100644 --- a/pkg/providers/ovh/ovh.go +++ b/pkg/providers/ovh/ovh.go @@ -39,6 +39,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } // OVH endpoint (default ovh-eu) endpoint := "ovh-eu" diff --git a/pkg/providers/scaleway/scaleway.go b/pkg/providers/scaleway/scaleway.go index 6277b52..6ea55b4 100644 --- a/pkg/providers/scaleway/scaleway.go +++ b/pkg/providers/scaleway/scaleway.go @@ -46,6 +46,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } client, err := scw.NewClient(scw.WithAuth(accessKey, accessToken)) if err != nil { diff --git a/pkg/providers/terraform/terraform.go b/pkg/providers/terraform/terraform.go index 9adcb09..be0eb62 100644 --- a/pkg/providers/terraform/terraform.go +++ b/pkg/providers/terraform/terraform.go @@ -46,6 +46,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } return &Provider{path: StatePathFile, id: id, services: services}, nil } diff --git a/pkg/providers/vercel/vercel.go b/pkg/providers/vercel/vercel.go index 48aecd4..47e5cf0 100644 --- a/pkg/providers/vercel/vercel.go +++ b/pkg/providers/vercel/vercel.go @@ -42,6 +42,11 @@ func New(options schema.OptionBlock) (*Provider, error) { services[s] = struct{}{} } } + if es, ok := options.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(es, ",") { + delete(services, strings.TrimSpace(s)) + } + } client := newAPIClient(newClientConfig{ Token: accessKey, diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 67ad620..5047219 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -212,18 +212,30 @@ type Options []OptionBlock // GetServiceNames returns the services from the options func (o Options) GetServiceNames() []string { - services := make([]string, 0) + return o.collectListValues("services") +} + +// GetExcludeServiceNames returns the excluded services from the options +func (o Options) GetExcludeServiceNames() []string { + return o.collectListValues("exclude_services") +} + +// collectListValues flattens a comma-separated option key across all blocks, +// trimming blanks. +func (o Options) collectListValues(key string) []string { + values := make([]string, 0) for _, option := range o { - if serviceNameList, ok := option["services"]; ok { - for _, serviceName := range strings.Split(serviceNameList, ",") { - trimmedServiceName := strings.TrimSpace(serviceName) - if trimmedServiceName != "" { - services = append(services, trimmedServiceName) - } + list, ok := option[key] + if !ok { + continue + } + for _, value := range strings.Split(list, ",") { + if trimmed := strings.TrimSpace(value); trimmed != "" { + values = append(values, trimmed) } } } - return services + return values } // OptionBlock is a single option on which operation is possible @@ -240,7 +252,7 @@ func (ob *OptionBlock) UnmarshalYAML(unmarshal func(interface{}) error) error { // Convert raw map to OptionBlock and handle special cases for key, value := range rawMap { switch key { - case "account_ids", "exclude_account_ids", "urls", "services", "project_ids", "exclude_project_ids": + case "account_ids", "exclude_account_ids", "urls", "services", "exclude_services", "project_ids", "exclude_project_ids": if valueArr, ok := value.([]interface{}); ok { var strArr []string for _, v := range valueArr { diff --git a/pkg/schema/schema_test.go b/pkg/schema/schema_test.go index 86d91c5..9e4da3e 100644 --- a/pkg/schema/schema_test.go +++ b/pkg/schema/schema_test.go @@ -75,6 +75,24 @@ func TestOptionBlockZeroPadsExcludeAccountIDs(t *testing.T) { require.Equal(t, "012345678901", value) } +func TestOptionBlockParsesExcludeServices(t *testing.T) { + data := ` +- provider: gcp + exclude_services: + - cloud-function + - gke +` + var options Options + err := yaml.Unmarshal([]byte(data), &options) + require.NoError(t, err) + require.Len(t, options, 1) + + value, ok := options[0].GetMetadata("exclude_services") + require.True(t, ok) + require.Equal(t, "cloud-function,gke", value) + require.Equal(t, []string{"cloud-function", "gke"}, options.GetExcludeServiceNames()) +} + func TestOptionBlockScalarFallback(t *testing.T) { data := ` - provider: aws From 2a64fd2298793ac9758c6fa32d2c13726cc29ebc Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 24 Jun 2026 21:47:31 +0200 Subject: [PATCH 2/4] address review --- PROVIDERS.md | 2 +- README.md | 2 +- internal/runner/options.go | 2 +- pkg/providers/alibaba/alibaba.go | 1 + pkg/providers/arvancloud/arvancloud.go | 1 + pkg/providers/aws/aws.go | 1 + pkg/providers/aws/aws_test.go | 8 +++++++- pkg/providers/azure/azure.go | 1 + pkg/providers/cloudflare/cloudflare.go | 1 + pkg/providers/custom/custom.go | 1 + pkg/providers/digitalocean/digitalocean.go | 1 + pkg/providers/dnssimple/dnssimple.go | 1 + pkg/providers/gcp/gcp.go | 2 ++ pkg/providers/heroku/heroku.go | 1 + pkg/providers/hetzner/hetzner.go | 1 + pkg/providers/k8s/kubernetes.go | 1 + pkg/providers/linode/linode.go | 1 + pkg/providers/namecheap/namecheap.go | 1 + pkg/providers/openstack/openstack.go | 1 + pkg/providers/scaleway/scaleway.go | 1 + pkg/providers/terraform/terraform.go | 1 + pkg/providers/vercel/vercel.go | 1 + 22 files changed, 29 insertions(+), 4 deletions(-) diff --git a/PROVIDERS.md b/PROVIDERS.md index b70e215..fa18154 100644 --- a/PROVIDERS.md +++ b/PROVIDERS.md @@ -1,6 +1,6 @@ # Providers -### Service filtering +## Service filtering Every provider enumerates all of its supported services by default. Two optional, additive keys narrow that set (also available on the CLI as `-s`/`--service` and `-es`/`--exclude-service`): diff --git a/README.md b/README.md index f5f3840..bc573f0 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ FILTERS: -id string[] display results for given ids (comma-separated) -host display only hostnames in results -ip display only ips in results - -s, -service value query and display results from given service (comma-separated)) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway) + -s, -service value query and display results from given service (comma-separated) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway) -es, -exclude-service value services to skip for a provider (comma-separated) -ep, -exclude-private exclude private ips in cli output diff --git a/internal/runner/options.go b/internal/runner/options.go index 028d895..0c3bdc2 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -85,7 +85,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Hosts, "host", false, "display only hostnames in results"), flagSet.BoolVar(&options.IPAddress, "ip", false, "display only ips in results"), flagSet.BoolVar(&options.ExtendedMetadata, "extended-metadata", false, "enable extended metadata for providers"), - flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated)) (default "+strings.Join(defaultServices, ",")+")", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated) (default "+strings.Join(defaultServices, ",")+")", goflags.CommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.ExcludeServices, "exclude-service", "es", nil, "services to skip for a provider (comma-separated)", goflags.CommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.ExcludePrivate, "exclude-private", "ep", false, "exclude private ips in cli output"), ) diff --git a/pkg/providers/alibaba/alibaba.go b/pkg/providers/alibaba/alibaba.go index e346665..a86e1d8 100644 --- a/pkg/providers/alibaba/alibaba.go +++ b/pkg/providers/alibaba/alibaba.go @@ -49,6 +49,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/arvancloud/arvancloud.go b/pkg/providers/arvancloud/arvancloud.go index 5338d05..a949538 100644 --- a/pkg/providers/arvancloud/arvancloud.go +++ b/pkg/providers/arvancloud/arvancloud.go @@ -43,6 +43,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/aws/aws.go b/pkg/providers/aws/aws.go index ac0ef9b..bcd4c6a 100644 --- a/pkg/providers/aws/aws.go +++ b/pkg/providers/aws/aws.go @@ -90,6 +90,7 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error { services := make(schema.ServiceMap) if ss, ok := block.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/aws/aws_test.go b/pkg/providers/aws/aws_test.go index 78ceac6..e833696 100644 --- a/pkg/providers/aws/aws_test.go +++ b/pkg/providers/aws/aws_test.go @@ -27,7 +27,13 @@ func TestParseOptionBlockExcludeServices(t *testing.T) { name: "exclude composes with services allowlist", block: schema.OptionBlock{"services": "ec2,s3,lambda", "exclude_services": "s3"}, wantPresent: []string{"ec2", "lambda"}, - wantAbsent: []string{"s3", "route53"}, + wantAbsent: []string{"s3"}, + }, + { + name: "whitespace is trimmed in both lists", + block: schema.OptionBlock{"services": "ec2, s3, lambda", "exclude_services": "s3, lambda"}, + wantPresent: []string{"ec2"}, + wantAbsent: []string{"s3", "lambda"}, }, { name: "unknown exclude value is ignored", diff --git a/pkg/providers/azure/azure.go b/pkg/providers/azure/azure.go index d72bd96..7c1e8bd 100644 --- a/pkg/providers/azure/azure.go +++ b/pkg/providers/azure/azure.go @@ -51,6 +51,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/cloudflare/cloudflare.go b/pkg/providers/cloudflare/cloudflare.go index 65c79fb..5d7fbaa 100644 --- a/pkg/providers/cloudflare/cloudflare.go +++ b/pkg/providers/cloudflare/cloudflare.go @@ -32,6 +32,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/custom/custom.go b/pkg/providers/custom/custom.go index 2376a87..dc0a5ac 100644 --- a/pkg/providers/custom/custom.go +++ b/pkg/providers/custom/custom.go @@ -91,6 +91,7 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error { services := make(schema.ServiceMap) if ss, ok := block.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/digitalocean/digitalocean.go b/pkg/providers/digitalocean/digitalocean.go index e3dd965..67a5a5c 100644 --- a/pkg/providers/digitalocean/digitalocean.go +++ b/pkg/providers/digitalocean/digitalocean.go @@ -33,6 +33,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/dnssimple/dnssimple.go b/pkg/providers/dnssimple/dnssimple.go index fb4acb7..693c4fe 100644 --- a/pkg/providers/dnssimple/dnssimple.go +++ b/pkg/providers/dnssimple/dnssimple.go @@ -60,6 +60,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/gcp/gcp.go b/pkg/providers/gcp/gcp.go index b7469ff..9ab340b 100644 --- a/pkg/providers/gcp/gcp.go +++ b/pkg/providers/gcp/gcp.go @@ -246,6 +246,7 @@ func newIndividualProvider(options schema.OptionBlock, id, JSONData string) (*Pr services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } @@ -670,6 +671,7 @@ func newOrganizationProvider(options schema.OptionBlock, id, JSONData, organizat services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/heroku/heroku.go b/pkg/providers/heroku/heroku.go index 6a5b3f8..6baee23 100644 --- a/pkg/providers/heroku/heroku.go +++ b/pkg/providers/heroku/heroku.go @@ -40,6 +40,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/hetzner/hetzner.go b/pkg/providers/hetzner/hetzner.go index 28d18fb..fda13b1 100644 --- a/pkg/providers/hetzner/hetzner.go +++ b/pkg/providers/hetzner/hetzner.go @@ -39,6 +39,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/k8s/kubernetes.go b/pkg/providers/k8s/kubernetes.go index 2a5ee01..ca421b8 100644 --- a/pkg/providers/k8s/kubernetes.go +++ b/pkg/providers/k8s/kubernetes.go @@ -71,6 +71,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/linode/linode.go b/pkg/providers/linode/linode.go index 4d3d401..91abd62 100644 --- a/pkg/providers/linode/linode.go +++ b/pkg/providers/linode/linode.go @@ -49,6 +49,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/namecheap/namecheap.go b/pkg/providers/namecheap/namecheap.go index 26d2fe8..0aeb480 100644 --- a/pkg/providers/namecheap/namecheap.go +++ b/pkg/providers/namecheap/namecheap.go @@ -59,6 +59,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/openstack/openstack.go b/pkg/providers/openstack/openstack.go index b392b10..fdcc290 100644 --- a/pkg/providers/openstack/openstack.go +++ b/pkg/providers/openstack/openstack.go @@ -89,6 +89,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/scaleway/scaleway.go b/pkg/providers/scaleway/scaleway.go index 6ea55b4..9011b89 100644 --- a/pkg/providers/scaleway/scaleway.go +++ b/pkg/providers/scaleway/scaleway.go @@ -36,6 +36,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/terraform/terraform.go b/pkg/providers/terraform/terraform.go index be0eb62..b4a0f51 100644 --- a/pkg/providers/terraform/terraform.go +++ b/pkg/providers/terraform/terraform.go @@ -36,6 +36,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } diff --git a/pkg/providers/vercel/vercel.go b/pkg/providers/vercel/vercel.go index 47e5cf0..bfff664 100644 --- a/pkg/providers/vercel/vercel.go +++ b/pkg/providers/vercel/vercel.go @@ -32,6 +32,7 @@ func New(options schema.OptionBlock) (*Provider, error) { services := make(schema.ServiceMap) if ss, ok := options.GetMetadata("services"); ok { for _, s := range strings.Split(ss, ",") { + s = strings.TrimSpace(s) if _, ok := supportedServicesMap[s]; ok { services[s] = struct{}{} } From 4d413265c83f068aea8bc1107c1ed6d9d33cd661 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 24 Jun 2026 21:57:37 +0200 Subject: [PATCH 3/4] centralize filtering --- build.err | 2 + pkg/providers/alibaba/alibaba.go | 25 +------- pkg/providers/arvancloud/arvancloud.go | 26 +-------- pkg/providers/aws/aws.go | 26 +-------- pkg/providers/azure/azure.go | 25 +------- pkg/providers/cloudflare/cloudflare.go | 26 +-------- pkg/providers/custom/custom.go | 36 +----------- pkg/providers/digitalocean/digitalocean.go | 25 +------- pkg/providers/dnssimple/dnssimple.go | 25 +------- pkg/providers/gcp/gcp.go | 24 +------- pkg/providers/heroku/heroku.go | 25 +------- pkg/providers/hetzner/hetzner.go | 25 +------- pkg/providers/k8s/kubernetes.go | 25 +------- pkg/providers/linode/linode.go | 25 +------- pkg/providers/namecheap/namecheap.go | 25 +------- pkg/providers/openstack/openstack.go | 25 +------- pkg/providers/ovh/ovh.go | 22 +------ pkg/providers/scaleway/scaleway.go | 25 +------- pkg/providers/terraform/terraform.go | 25 +------- pkg/providers/vercel/vercel.go | 25 +------- pkg/schema/schema.go | 37 ++++++++++++ pkg/schema/schema_test.go | 68 ++++++++++++++++++++++ 22 files changed, 127 insertions(+), 465 deletions(-) create mode 100644 build.err diff --git a/build.err b/build.err new file mode 100644 index 0000000..011cedb --- /dev/null +++ b/build.err @@ -0,0 +1,2 @@ +# github.com/projectdiscovery/cloudlist/pkg/providers/custom +pkg/providers/custom/custom.go:87:2: declared and not used: services diff --git a/pkg/providers/alibaba/alibaba.go b/pkg/providers/alibaba/alibaba.go index a86e1d8..9177e53 100644 --- a/pkg/providers/alibaba/alibaba.go +++ b/pkg/providers/alibaba/alibaba.go @@ -2,7 +2,6 @@ package alibaba import ( "context" - "strings" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -42,29 +41,7 @@ func New(options schema.OptionBlock) (*Provider, error) { id, _ := options.GetMetadata("id") provider := &Provider{id: id} - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) provider.services = services if services.Has("instance") { diff --git a/pkg/providers/arvancloud/arvancloud.go b/pkg/providers/arvancloud/arvancloud.go index a949538..e13f038 100644 --- a/pkg/providers/arvancloud/arvancloud.go +++ b/pkg/providers/arvancloud/arvancloud.go @@ -2,7 +2,6 @@ package arvancloud import ( "context" - "strings" r1c "git.arvancloud.ir/arvancloud/cdn-go-sdk" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -35,30 +34,7 @@ func New(options schema.OptionBlock) (*Provider, error) { // Construct a new API object api := r1c.NewAPIClient(configuration) - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: api, services: services}, nil } diff --git a/pkg/providers/aws/aws.go b/pkg/providers/aws/aws.go index bcd4c6a..61e1daf 100644 --- a/pkg/providers/aws/aws.go +++ b/pkg/providers/aws/aws.go @@ -83,31 +83,7 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error { p.OrgDiscoveryRoleArn = orgRoleArn } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := block.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - // if no services provided from -service flag, includes all services - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := block.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } - p.Services = services + p.Services = block.ResolveServices(Services) if extendedMetadata, ok := block.GetMetadata("extended_metadata"); ok { p.ExtendedMetadata = extendedMetadata == "true" diff --git a/pkg/providers/azure/azure.go b/pkg/providers/azure/azure.go index 7c1e8bd..9622122 100644 --- a/pkg/providers/azure/azure.go +++ b/pkg/providers/azure/azure.go @@ -3,7 +3,6 @@ package azure import ( "context" "fmt" - "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions" @@ -44,29 +43,7 @@ func New(options schema.OptionBlock) (*Provider, error) { } // Parse services - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) provider := &Provider{ Credential: credential, // Track 2: use credential instead of authorizer diff --git a/pkg/providers/cloudflare/cloudflare.go b/pkg/providers/cloudflare/cloudflare.go index 5d7fbaa..4a70def 100644 --- a/pkg/providers/cloudflare/cloudflare.go +++ b/pkg/providers/cloudflare/cloudflare.go @@ -3,7 +3,6 @@ package cloudflare import ( "context" "fmt" - "strings" "github.com/cloudflare/cloudflare-go" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -24,30 +23,7 @@ type Provider struct { func New(options schema.OptionBlock) (*Provider, error) { id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) // Parse extended metadata option extendedMetadata := false diff --git a/pkg/providers/custom/custom.go b/pkg/providers/custom/custom.go index dc0a5ac..a29b8cc 100644 --- a/pkg/providers/custom/custom.go +++ b/pkg/providers/custom/custom.go @@ -43,17 +43,8 @@ func New(block schema.OptionBlock) (*Provider, error) { return nil, err } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - - services := make(schema.ServiceMap) - for _, s := range Services { - services[s] = struct{}{} - } client := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle) - return &Provider{client: client, id: options.Id, urlList: options.URLs, headerList: options.Headers, services: services}, nil + return &Provider{client: client, id: options.Id, urlList: options.URLs, headerList: options.Headers, services: options.Services}, nil } // Name returns the name of the provider @@ -84,30 +75,7 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error { p.Id, _ = block.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := block.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - // if no services provided from -service flag, includes all services - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := block.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + p.Services = block.ResolveServices(Services) np, err := networkpolicy.New(networkpolicy.DefaultOptions) if err != nil { diff --git a/pkg/providers/digitalocean/digitalocean.go b/pkg/providers/digitalocean/digitalocean.go index 67a5a5c..6cbfd83 100644 --- a/pkg/providers/digitalocean/digitalocean.go +++ b/pkg/providers/digitalocean/digitalocean.go @@ -2,7 +2,6 @@ package digitalocean import ( "context" - "strings" "github.com/digitalocean/godo" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -26,29 +25,7 @@ func New(options schema.OptionBlock) (*Provider, error) { } id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) // Check for extended metadata option extendedMetadata := false diff --git a/pkg/providers/dnssimple/dnssimple.go b/pkg/providers/dnssimple/dnssimple.go index 693c4fe..e7bcc25 100644 --- a/pkg/providers/dnssimple/dnssimple.go +++ b/pkg/providers/dnssimple/dnssimple.go @@ -3,7 +3,6 @@ package dnssimple import ( "context" "fmt" - "strings" "github.com/dnsimple/dnsimple-go/dnsimple" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -53,29 +52,7 @@ func New(options schema.OptionBlock) (*Provider, error) { client := dnsimple.NewClient(dnsimple.StaticTokenHTTPClient(context.Background(), token)) // Configure services - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) provider := &Provider{ id: id, diff --git a/pkg/providers/gcp/gcp.go b/pkg/providers/gcp/gcp.go index 9ab340b..5f26e55 100644 --- a/pkg/providers/gcp/gcp.go +++ b/pkg/providers/gcp/gcp.go @@ -239,29 +239,7 @@ func newIndividualProvider(options schema.OptionBlock, id, JSONData string) (*Pr provider.extendedMetadata = extendedMetadata == "true" } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) provider.services = services configuredProjects := getProjectIDsFromOptions(options) diff --git a/pkg/providers/heroku/heroku.go b/pkg/providers/heroku/heroku.go index 6baee23..c55564e 100644 --- a/pkg/providers/heroku/heroku.go +++ b/pkg/providers/heroku/heroku.go @@ -2,7 +2,6 @@ package heroku import ( "context" - "strings" heroku "github.com/heroku/heroku-go/v5" @@ -33,29 +32,7 @@ func New(options schema.OptionBlock) (*Provider, error) { heroku.DefaultTransport.BearerToken = token - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: heroku.NewService(heroku.DefaultClient), services: services}, nil } diff --git a/pkg/providers/hetzner/hetzner.go b/pkg/providers/hetzner/hetzner.go index fda13b1..f7bd07f 100644 --- a/pkg/providers/hetzner/hetzner.go +++ b/pkg/providers/hetzner/hetzner.go @@ -2,7 +2,6 @@ package hetzner import ( "context" - "strings" hetzner "github.com/hetznercloud/hcloud-go/hcloud" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -32,29 +31,7 @@ func New(options schema.OptionBlock) (*Provider, error) { id, _ := options.GetMetadata("id") opts := hetzner.WithToken(token) - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: hetzner.NewClient(opts), services: services}, nil } diff --git a/pkg/providers/k8s/kubernetes.go b/pkg/providers/k8s/kubernetes.go index ca421b8..e5ba2fe 100644 --- a/pkg/providers/k8s/kubernetes.go +++ b/pkg/providers/k8s/kubernetes.go @@ -4,7 +4,6 @@ import ( "context" "encoding/base64" "fmt" - "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" "github.com/projectdiscovery/utils/errkit" @@ -64,29 +63,7 @@ func New(options schema.OptionBlock) (*Provider, error) { return nil, errkit.Wrap(err, "could not create kubernetes clientset") } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) var providerExtendedMetadata bool if extendedMetadata, ok := options.GetMetadata("extended_metadata"); ok { providerExtendedMetadata = extendedMetadata == "true" diff --git a/pkg/providers/linode/linode.go b/pkg/providers/linode/linode.go index 91abd62..b675b6d 100644 --- a/pkg/providers/linode/linode.go +++ b/pkg/providers/linode/linode.go @@ -3,7 +3,6 @@ package linode import ( "context" "net/http" - "strings" "github.com/linode/linodego" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -42,29 +41,7 @@ func New(options schema.OptionBlock) (*Provider, error) { client := linodego.NewClient(oc) - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: &client, services: services}, nil } diff --git a/pkg/providers/namecheap/namecheap.go b/pkg/providers/namecheap/namecheap.go index 0aeb480..96f51df 100644 --- a/pkg/providers/namecheap/namecheap.go +++ b/pkg/providers/namecheap/namecheap.go @@ -2,7 +2,6 @@ package namecheap import ( "context" - "strings" "github.com/namecheap/go-namecheap-sdk/v2/namecheap" @@ -52,29 +51,7 @@ func New(options schema.OptionBlock) (*Provider, error) { UseSandbox: false, } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: namecheap.NewClient(&clientOptions), services: services}, nil } diff --git a/pkg/providers/openstack/openstack.go b/pkg/providers/openstack/openstack.go index fdcc290..eecb071 100644 --- a/pkg/providers/openstack/openstack.go +++ b/pkg/providers/openstack/openstack.go @@ -2,7 +2,6 @@ package openstack import ( "context" - "strings" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack" @@ -82,29 +81,7 @@ func New(options schema.OptionBlock) (*Provider, error) { return nil, err } - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) return &Provider{id: id, client: client, services: services}, nil } diff --git a/pkg/providers/ovh/ovh.go b/pkg/providers/ovh/ovh.go index 9eda436..ece059c 100644 --- a/pkg/providers/ovh/ovh.go +++ b/pkg/providers/ovh/ovh.go @@ -3,7 +3,6 @@ package ovh import ( "context" "net/http" - "strings" "time" "github.com/ovh/go-ovh/ovh" @@ -24,26 +23,7 @@ func New(options schema.OptionBlock) (*Provider, error) { id, _ := options.GetMetadata("id") // service selection - supported := map[string]struct{}{"dns": {}} - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supported[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) // OVH endpoint (default ovh-eu) endpoint := "ovh-eu" diff --git a/pkg/providers/scaleway/scaleway.go b/pkg/providers/scaleway/scaleway.go index 9011b89..4c9bc04 100644 --- a/pkg/providers/scaleway/scaleway.go +++ b/pkg/providers/scaleway/scaleway.go @@ -2,7 +2,6 @@ package scaleway import ( "context" - "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" @@ -29,29 +28,7 @@ func New(options schema.OptionBlock) (*Provider, error) { return nil, &schema.ErrNoSuchKey{Name: apiAccessToken} } id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) client, err := scw.NewClient(scw.WithAuth(accessKey, accessToken)) if err != nil { diff --git a/pkg/providers/terraform/terraform.go b/pkg/providers/terraform/terraform.go index b4a0f51..80ec50a 100644 --- a/pkg/providers/terraform/terraform.go +++ b/pkg/providers/terraform/terraform.go @@ -2,7 +2,6 @@ package terraform import ( "context" - "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" ) @@ -29,29 +28,7 @@ func New(options schema.OptionBlock) (*Provider, error) { } id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) return &Provider{path: StatePathFile, id: id, services: services}, nil } diff --git a/pkg/providers/vercel/vercel.go b/pkg/providers/vercel/vercel.go index bfff664..1427dc6 100644 --- a/pkg/providers/vercel/vercel.go +++ b/pkg/providers/vercel/vercel.go @@ -2,7 +2,6 @@ package vercel import ( "context" - "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" ) @@ -25,29 +24,7 @@ func New(options schema.OptionBlock) (*Provider, error) { teamID, _ := options.GetMetadata(apiTeamID) id, _ := options.GetMetadata("id") - supportedServicesMap := make(map[string]struct{}) - for _, s := range Services { - supportedServicesMap[s] = struct{}{} - } - services := make(schema.ServiceMap) - if ss, ok := options.GetMetadata("services"); ok { - for _, s := range strings.Split(ss, ",") { - s = strings.TrimSpace(s) - if _, ok := supportedServicesMap[s]; ok { - services[s] = struct{}{} - } - } - } - if len(services) == 0 { - for _, s := range Services { - services[s] = struct{}{} - } - } - if es, ok := options.GetMetadata("exclude_services"); ok { - for _, s := range strings.Split(es, ",") { - delete(services, strings.TrimSpace(s)) - } - } + services := options.ResolveServices(Services) client := newAPIClient(newClientConfig{ Token: accessKey, diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 5047219..38ec6ad 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -308,6 +308,43 @@ func (o OptionBlock) GetMetadata(key string) (string, bool) { return data, true } +// ResolveServices computes the effective service set for a provider from the +// option block. It starts from the `services` allowlist (falling back to all +// supported services when the allowlist is empty), then removes any +// `exclude_services` entries. Entries are whitespace-trimmed and values that +// are not in supported are ignored. +func (o OptionBlock) ResolveServices(supported []string) ServiceMap { + supportedSet := make(map[string]struct{}, len(supported)) + for _, s := range supported { + supportedSet[s] = struct{}{} + } + + services := make(ServiceMap) + if allow, ok := o.GetMetadata("services"); ok { + for _, s := range strings.Split(allow, ",") { + s = strings.TrimSpace(s) + if _, ok := supportedSet[s]; ok { + services[s] = struct{}{} + } + } + } + + // default to all supported services when no allowlist is provided + if len(services) == 0 { + for _, s := range supported { + services[s] = struct{}{} + } + } + + if exclude, ok := o.GetMetadata("exclude_services"); ok { + for _, s := range strings.Split(exclude, ",") { + delete(services, strings.TrimSpace(s)) + } + } + + return services +} + type ServiceMap map[string]struct{} func (s ServiceMap) Has(service string) bool { diff --git a/pkg/schema/schema_test.go b/pkg/schema/schema_test.go index 9e4da3e..ebb16f7 100644 --- a/pkg/schema/schema_test.go +++ b/pkg/schema/schema_test.go @@ -93,6 +93,74 @@ func TestOptionBlockParsesExcludeServices(t *testing.T) { require.Equal(t, []string{"cloud-function", "gke"}, options.GetExcludeServiceNames()) } +func TestResolveServices(t *testing.T) { + supported := []string{"ec2", "s3", "lambda", "route53"} + + tests := []struct { + name string + block OptionBlock + wantPresent []string + wantAbsent []string + }{ + { + name: "no options defaults to all supported", + block: OptionBlock{}, + wantPresent: supported, + }, + { + name: "allowlist keeps only listed services", + block: OptionBlock{"services": "ec2,s3"}, + wantPresent: []string{"ec2", "s3"}, + wantAbsent: []string{"lambda", "route53"}, + }, + { + name: "exclude drops from default-all set", + block: OptionBlock{"exclude_services": "s3,route53"}, + wantPresent: []string{"ec2", "lambda"}, + wantAbsent: []string{"s3", "route53"}, + }, + { + name: "exclude composes with allowlist", + block: OptionBlock{"services": "ec2,s3,lambda", "exclude_services": "s3"}, + wantPresent: []string{"ec2", "lambda"}, + wantAbsent: []string{"s3"}, + }, + { + name: "whitespace is trimmed in both lists", + block: OptionBlock{"services": "ec2, s3, lambda", "exclude_services": "s3, lambda"}, + wantPresent: []string{"ec2"}, + wantAbsent: []string{"s3", "lambda"}, + }, + { + name: "unknown allowlist values are ignored and fall back to all", + block: OptionBlock{"services": "not-a-service"}, + wantPresent: supported, + }, + { + name: "unknown exclude values are ignored", + block: OptionBlock{"exclude_services": "not-a-service"}, + wantPresent: supported, + }, + { + name: "excluding every service yields an empty set", + block: OptionBlock{"exclude_services": "ec2,s3,lambda,route53"}, + wantAbsent: supported, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + services := tt.block.ResolveServices(supported) + for _, s := range tt.wantPresent { + require.Truef(t, services.Has(s), "expected service %q to be present", s) + } + for _, s := range tt.wantAbsent { + require.Falsef(t, services.Has(s), "expected service %q to be absent", s) + } + }) + } +} + func TestOptionBlockScalarFallback(t *testing.T) { data := ` - provider: aws From ea386f4bbaef395feba2ee8a977d6b1ff7745add Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 24 Jun 2026 21:57:51 +0200 Subject: [PATCH 4/4] remove artifact --- build.err | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 build.err diff --git a/build.err b/build.err deleted file mode 100644 index 011cedb..0000000 --- a/build.err +++ /dev/null @@ -1,2 +0,0 @@ -# github.com/projectdiscovery/cloudlist/pkg/providers/custom -pkg/providers/custom/custom.go:87:2: declared and not used: services