diff --git a/duckduckgo/parse_html.go b/duckduckgo/parse_html.go index 9d18d20..decc415 100644 --- a/duckduckgo/parse_html.go +++ b/duckduckgo/parse_html.go @@ -55,7 +55,7 @@ func parseDDGDocument(doc *goquery.Document) []core.SearchResult { } desc := firstText(item, Selectors.Desc...) - isAd := ddgSelectionHasAdMarker(item) + isAd := ddgSelectionHasAdMarker(item) || isDDGAdURL(href) resultRank, absoluteRank := rank.Next(isAd) results = append(results, core.SearchResult{ diff --git a/duckduckgo/parse_html_test.go b/duckduckgo/parse_html_test.go index 6e9a4da..213163d 100644 --- a/duckduckgo/parse_html_test.go +++ b/duckduckgo/parse_html_test.go @@ -129,3 +129,55 @@ func TestParseDDGHTMLAdsDoNotConsumeOrganicRank(t *testing.T) { t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank) } } + +// An ad card can come without any ad badge; its y.js click-tracking link still +// marks it as an ad. +func TestParseDDGHTMLAdWithoutBadgeIsDetectedByLink(t *testing.T) { + t.Parallel() + + html := ` +
+

GoDaddy Domains

+
Paid snippet
+
+
+

Organic One

+
Organic snippet one
+
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + if !results[0].Ad { + t.Fatalf("y.js result not marked as ad: %+v", results[0]) + } + if results[1].Ad || results[1].Rank != 1 { + t.Fatalf("organic result: Ad = %v, Rank = %d, want false and 1", results[1].Ad, results[1].Rank) + } +} + +func TestIsDDGAdURL(t *testing.T) { + t.Parallel() + + tests := []struct { + url string + want bool + }{ + {"https://duckduckgo.com/y.js?ad_domain=godaddy.com&ad_provider=bingv7aa", true}, + {"https://duckduckgo.com/y.js?u3=https%3A%2F%2Fwww.bing.com%2Faclick", true}, + {"/y.js?ad_domain=example.com", true}, + {"https://organic.example.com/one", false}, + {"https://example.com/y.js", false}, + {"https://duckduckgo.com/?q=y.js", false}, + {"", false}, + } + for _, tt := range tests { + if got := isDDGAdURL(tt.url); got != tt.want { + t.Errorf("isDDGAdURL(%q) = %v, want %v", tt.url, got, tt.want) + } + } +} diff --git a/duckduckgo/search.go b/duckduckgo/search.go index dc9184b..8830f28 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -113,7 +113,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se desc := core.FirstNonEmptyText(r, Selectors.Desc...) // Check if it's an ad - isAd := ddgElementHasAdMarker(r) + isAd := ddgElementHasAdMarker(r) || isDDGAdURL(hrefStr) resultRank, absoluteRank := rank.Next(isAd) result := core.SearchResult{ diff --git a/duckduckgo/url.go b/duckduckgo/url.go index 6877374..3461131 100644 --- a/duckduckgo/url.go +++ b/duckduckgo/url.go @@ -177,3 +177,18 @@ func BuildImageURL(q core.Query) (string, error) { base.RawQuery = params.Encode() return base.String(), nil } + +// isDDGAdURL reports whether href is DuckDuckGo's ad click-tracking link +// (/y.js on duckduckgo.com, absolute or relative). Ad cards keep this link +// even when they carry no ad badge. +func isDDGAdURL(href string) bool { + u, err := url.Parse(strings.TrimSpace(href)) + if err != nil { + return false + } + host := strings.ToLower(u.Hostname()) + if host != "" && host != "duckduckgo.com" && !strings.HasSuffix(host, ".duckduckgo.com") { + return false + } + return u.Path == "/y.js" +}