initial

README.md

@@ -0,0 +1,3 @@
+# Assetfinder
+
+Domain recon tool

bufferoverrun.go

@@ -0,0 +1,32 @@
+package assetfinder
+
+import (
+	"fmt"
+	"strings"
+)
+
+type BufferOverrun struct{}
+
+func (BufferOverrun) FetchSubDomains(domain string) ([]string, error) {
+	out := make([]string, 0)
+
+	fetchURL := fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain)
+
+	wrapper := struct {
+		Records []string `json:"FDNS_A"`
+	}{}
+	err := fetchJSON(fetchURL, &wrapper)
+	if err != nil {
+		return out, err
+	}
+
+	for _, r := range wrapper.Records {
+		parts := strings.SplitN(r, ",", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		out = append(out, parts[1])
+	}
+
+	return out, nil
+}

certspotter.go

@@ -0,0 +1,25 @@
+package assetfinder
+
+import "fmt"
+
+type CertSpotter struct{}
+
+func (CertSpotter) FetchSubDomains(domain string) ([]string, error) {
+	out := make([]string, 0)
+
+	fetchURL := fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain)
+
+	wrapper := []struct {
+		DNSNames []string `json:"dns_names"`
+	}{}
+	err := fetchJSON(fetchURL, &wrapper)
+	if err != nil {
+		return out, err
+	}
+
+	for _, w := range wrapper {
+		out = append(out, w.DNSNames...)
+	}
+
+	return out, nil
+}

cmd/assetfinder/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/cubixle/assetfinder"
+)
+
+func main() {
+	// subsOnly := flag.Bool("subs-only", false, "Only search for subdomains.")
+	disableStatusCheck := flag.Bool("status", false, "Enable/Disable checking the status code for each sub domain found.")
+	checkHTTPS := flag.Bool("https", true, "Enable/Disable the checking of status of https on a domain.")
+	outfile := flag.String("output", "", "Where to save results to.")
+	verbose := flag.Bool("verbose", false, "Turns on verbose logging. Mainly used for debugging development.")
+
+	flag.Parse()
+
+	domain := flag.Arg(0)
+	if domain == "" {
+		log.Fatal("no domain given")
+	}
+	if err := assetfinder.Scanner(*verbose, *checkHTTPS, *disableStatusCheck, *outfile, domain); err != nil {
+		log.Fatal(err)
+	}
+}

crtsh.go

@@ -0,0 +1,39 @@
+package assetfinder
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+type CrtShResult struct {
+	Name string `json:"name_value"`
+}
+
+type CrtSh struct{}
+
+func (CrtSh) FetchSubDomains(domain string) ([]string, error) {
+	var results []CrtShResult
+
+	resp, err := http.Get(
+		fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain),
+	)
+	if err != nil {
+		return []string{}, err
+	}
+	defer resp.Body.Close()
+
+	output := make([]string, 0)
+
+	body, _ := ioutil.ReadAll(resp.Body)
+
+	if err := json.Unmarshal(body, &results); err != nil {
+		return []string{}, err
+	}
+
+	for _, res := range results {
+		output = append(output, res.Name)
+	}
+	return output, nil
+}

go.mod

@@ -0,0 +1,3 @@
+module github.com/cubixle/assetfinder
+
+go 1.14

scanner.go

@@ -0,0 +1,126 @@
+package assetfinder
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/url"
+	"strings"
+)
+
+func Scanner(verboseLogging, checkHTTPS, disableStatusCheck bool, outputFile, domain string) error {
+	results := []string{}
+	for _, source := range LoadedSources {
+		subdomains, err := source.FetchSubDomains(domain)
+		if err != nil {
+			continue
+		}
+		results = append(results, subdomains...)
+	}
+	sortedDomains := sortDomains(results)
+
+	ress := []Result{}
+	if !disableStatusCheck {
+		for _, r := range sortedDomains {
+			// http
+			u, err := url.Parse("http://" + r)
+			if err != nil {
+				if verboseLogging {
+					log.Println(err)
+				}
+				continue
+			}
+
+			re := fetchStatus(u.String())
+			ress = append(ress, re)
+
+			if checkHTTPS && re.StatusCode != 0 {
+				u.Scheme = "https"
+				re := fetchStatus(u.String())
+				ress = append(ress, re)
+			}
+		}
+	} else {
+		ress = convertToResults(results)
+	}
+
+	if outputFile == "" {
+		for _, r := range ress {
+			if r.StatusCode == 0 || r.Error != nil {
+				continue
+			}
+			log.Printf("| %d | %s\n", r.StatusCode, r.Domain)
+		}
+		return nil
+	}
+
+	return writeFile(outputFile, ress)
+}
+
+func writeFile(outputFile string, res []Result) error {
+	resBytes := []byte("domain, status code")
+	for _, r := range res {
+		if r.StatusCode == 0 || r.Error != nil {
+			continue
+		}
+		resBytes = append(resBytes, []byte(fmt.Sprintf("%s, %d\n", r.Domain, r.StatusCode))...)
+	}
+
+	err := ioutil.WriteFile(outputFile, resBytes, 0775)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func sortDomains(domains []string) []string {
+	encountered := map[string]string{}
+	fixedList := []string{}
+	for _, d := range domains {
+		if strings.Contains(d, "\n") {
+			for _, d := range strings.Split(d, "\n") {
+				d = cleanDomain(d)
+				fixedList = append(fixedList, d)
+			}
+		} else {
+			d = cleanDomain(d)
+			fixedList = append(fixedList, d)
+		}
+	}
+
+	// dedup
+	newList := []string{}
+	for _, d := range fixedList {
+		_, ok := encountered[d]
+		if !ok {
+			encountered[d] = d
+			newList = append(newList, d)
+		}
+	}
+	return newList
+}
+
+func convertToResults(strRes []string) []Result {
+	ress := []Result{}
+	for _, r := range strRes {
+		ress = append(ress, Result{
+			Domain: r,
+		})
+	}
+	return ress
+}
+
+func cleanDomain(d string) string {
+	d = strings.ToLower(d)
+	if len(d) < 2 {
+		return d
+	}
+	if d[0] == '*' || d[0] == '%' {
+		d = d[1:]
+	}
+
+	if d[0] == '.' {
+		d = d[1:]
+	}
+	return d
+}

sources.go

@@ -0,0 +1,67 @@
+package assetfinder
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+)
+
+var LoadedSources = []Source{
+	BufferOverrun{},
+	CertSpotter{},
+	CrtSh{},
+	URLScan{},
+}
+
+// Source is an interface that defines the methods required by a source.
+type Source interface {
+	FetchSubDomains(domain string) ([]string, error)
+}
+
+// Result is the basic type that should be return from the sources.Fetch method.
+type Result struct {
+	Domain     string
+	StatusCode int
+	Error      error
+}
+
+func fetchJSON(url string, i interface{}) error {
+	client := httpClient()
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil
+	}
+
+	defer resp.Body.Close()
+	dec := json.NewDecoder(resp.Body)
+	return dec.Decode(i)
+}
+
+func fetchStatus(url string) Result {
+	res := Result{Domain: url}
+	request, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		res.Error = err
+		return res
+	}
+
+	client := httpClient()
+	resp, err := client.Do(request)
+	if err != nil {
+		res.Error = err
+		return res
+	}
+	res.StatusCode = resp.StatusCode
+	return res
+}
+
+func httpClient() http.Client {
+	return http.Client{
+		Timeout: time.Duration(5 * time.Second),
+	}
+}

urlscan.go

@@ -0,0 +1,52 @@
+package assetfinder
+
+import (
+	"fmt"
+	"net/url"
+)
+
+type URLScan struct{}
+
+type URLScanResults struct {
+	Results []struct {
+		Task struct {
+			URL string `json:"url"`
+		} `json:"task"`
+
+		Page struct {
+			URL string `json:"url"`
+		} `json:"page"`
+	} `json:"results"`
+}
+
+func (URLScan) FetchSubDomains(domain string) ([]string, error) {
+	resp := URLScanResults{}
+	err := fetchJSON(
+		fmt.Sprintf("https://urlscan.io/api/v1/search/?q=domain:%s", domain),
+		resp,
+	)
+	if err != nil {
+		return []string{}, err
+	}
+	output := make([]string, 0)
+
+	for _, r := range resp.Results {
+		u, err := url.Parse(r.Task.URL)
+		if err != nil {
+			continue
+		}
+
+		output = append(output, u.Hostname())
+	}
+
+	for _, r := range resp.Results {
+		u, err := url.Parse(r.Page.URL)
+		if err != nil {
+			continue
+		}
+
+		output = append(output, u.Hostname())
+	}
+
+	return output, nil
+}