commit f6be925acfe28681be40063c6e02a479d87f4f29 Author: cubixle Date: Fri Jul 24 22:37:25 2020 +0100 initial diff --git a/README.md b/README.md new file mode 100644 index 0000000..15861ab --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Assetfinder + +Domain recon tool \ No newline at end of file diff --git a/bufferoverrun.go b/bufferoverrun.go new file mode 100644 index 0000000..7b70dec --- /dev/null +++ b/bufferoverrun.go @@ -0,0 +1,32 @@ +package assetfinder + +import ( + "fmt" + "strings" +) + +type BufferOverrun struct{} + +func (BufferOverrun) FetchSubDomains(domain string) ([]string, error) { + out := make([]string, 0) + + fetchURL := fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain) + + wrapper := struct { + Records []string `json:"FDNS_A"` + }{} + err := fetchJSON(fetchURL, &wrapper) + if err != nil { + return out, err + } + + for _, r := range wrapper.Records { + parts := strings.SplitN(r, ",", 2) + if len(parts) != 2 { + continue + } + out = append(out, parts[1]) + } + + return out, nil +} diff --git a/certspotter.go b/certspotter.go new file mode 100644 index 0000000..6aa0cb0 --- /dev/null +++ b/certspotter.go @@ -0,0 +1,25 @@ +package assetfinder + +import "fmt" + +type CertSpotter struct{} + +func (CertSpotter) FetchSubDomains(domain string) ([]string, error) { + out := make([]string, 0) + + fetchURL := fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain) + + wrapper := []struct { + DNSNames []string `json:"dns_names"` + }{} + err := fetchJSON(fetchURL, &wrapper) + if err != nil { + return out, err + } + + for _, w := range wrapper { + out = append(out, w.DNSNames...) + } + + return out, nil +} diff --git a/cmd/assetfinder/main.go b/cmd/assetfinder/main.go new file mode 100644 index 0000000..511b6b9 --- /dev/null +++ b/cmd/assetfinder/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "flag" + "log" + + "github.com/cubixle/assetfinder" +) + +func main() { + // subsOnly := flag.Bool("subs-only", false, "Only search for subdomains.") + disableStatusCheck := flag.Bool("status", false, "Enable/Disable checking the status code for each sub domain found.") + checkHTTPS := flag.Bool("https", true, "Enable/Disable the checking of status of https on a domain.") + outfile := flag.String("output", "", "Where to save results to.") + verbose := flag.Bool("verbose", false, "Turns on verbose logging. Mainly used for debugging development.") + + flag.Parse() + + domain := flag.Arg(0) + if domain == "" { + log.Fatal("no domain given") + } + if err := assetfinder.Scanner(*verbose, *checkHTTPS, *disableStatusCheck, *outfile, domain); err != nil { + log.Fatal(err) + } +} diff --git a/crtsh.go b/crtsh.go new file mode 100644 index 0000000..8a8b7e4 --- /dev/null +++ b/crtsh.go @@ -0,0 +1,39 @@ +package assetfinder + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +type CrtShResult struct { + Name string `json:"name_value"` +} + +type CrtSh struct{} + +func (CrtSh) FetchSubDomains(domain string) ([]string, error) { + var results []CrtShResult + + resp, err := http.Get( + fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain), + ) + if err != nil { + return []string{}, err + } + defer resp.Body.Close() + + output := make([]string, 0) + + body, _ := ioutil.ReadAll(resp.Body) + + if err := json.Unmarshal(body, &results); err != nil { + return []string{}, err + } + + for _, res := range results { + output = append(output, res.Name) + } + return output, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a347e55 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/cubixle/assetfinder + +go 1.14 diff --git a/scanner.go b/scanner.go new file mode 100644 index 0000000..5e94c0e --- /dev/null +++ b/scanner.go @@ -0,0 +1,126 @@ +package assetfinder + +import ( + "fmt" + "io/ioutil" + "log" + "net/url" + "strings" +) + +func Scanner(verboseLogging, checkHTTPS, disableStatusCheck bool, outputFile, domain string) error { + results := []string{} + for _, source := range LoadedSources { + subdomains, err := source.FetchSubDomains(domain) + if err != nil { + continue + } + results = append(results, subdomains...) + } + sortedDomains := sortDomains(results) + + ress := []Result{} + if !disableStatusCheck { + for _, r := range sortedDomains { + // http + u, err := url.Parse("http://" + r) + if err != nil { + if verboseLogging { + log.Println(err) + } + continue + } + + re := fetchStatus(u.String()) + ress = append(ress, re) + + if checkHTTPS && re.StatusCode != 0 { + u.Scheme = "https" + re := fetchStatus(u.String()) + ress = append(ress, re) + } + } + } else { + ress = convertToResults(results) + } + + if outputFile == "" { + for _, r := range ress { + if r.StatusCode == 0 || r.Error != nil { + continue + } + log.Printf("| %d | %s\n", r.StatusCode, r.Domain) + } + return nil + } + + return writeFile(outputFile, ress) +} + +func writeFile(outputFile string, res []Result) error { + resBytes := []byte("domain, status code") + for _, r := range res { + if r.StatusCode == 0 || r.Error != nil { + continue + } + resBytes = append(resBytes, []byte(fmt.Sprintf("%s, %d\n", r.Domain, r.StatusCode))...) + } + + err := ioutil.WriteFile(outputFile, resBytes, 0775) + if err != nil { + return err + } + return nil +} + +func sortDomains(domains []string) []string { + encountered := map[string]string{} + fixedList := []string{} + for _, d := range domains { + if strings.Contains(d, "\n") { + for _, d := range strings.Split(d, "\n") { + d = cleanDomain(d) + fixedList = append(fixedList, d) + } + } else { + d = cleanDomain(d) + fixedList = append(fixedList, d) + } + } + + // dedup + newList := []string{} + for _, d := range fixedList { + _, ok := encountered[d] + if !ok { + encountered[d] = d + newList = append(newList, d) + } + } + return newList +} + +func convertToResults(strRes []string) []Result { + ress := []Result{} + for _, r := range strRes { + ress = append(ress, Result{ + Domain: r, + }) + } + return ress +} + +func cleanDomain(d string) string { + d = strings.ToLower(d) + if len(d) < 2 { + return d + } + if d[0] == '*' || d[0] == '%' { + d = d[1:] + } + + if d[0] == '.' { + d = d[1:] + } + return d +} diff --git a/sources.go b/sources.go new file mode 100644 index 0000000..acaa353 --- /dev/null +++ b/sources.go @@ -0,0 +1,67 @@ +package assetfinder + +import ( + "encoding/json" + "net/http" + "time" +) + +var LoadedSources = []Source{ + BufferOverrun{}, + CertSpotter{}, + CrtSh{}, + URLScan{}, +} + +// Source is an interface that defines the methods required by a source. +type Source interface { + FetchSubDomains(domain string) ([]string, error) +} + +// Result is the basic type that should be return from the sources.Fetch method. +type Result struct { + Domain string + StatusCode int + Error error +} + +func fetchJSON(url string, i interface{}) error { + client := httpClient() + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err + } + + resp, err := client.Do(req) + if err != nil { + return nil + } + + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + return dec.Decode(i) +} + +func fetchStatus(url string) Result { + res := Result{Domain: url} + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + res.Error = err + return res + } + + client := httpClient() + resp, err := client.Do(request) + if err != nil { + res.Error = err + return res + } + res.StatusCode = resp.StatusCode + return res +} + +func httpClient() http.Client { + return http.Client{ + Timeout: time.Duration(5 * time.Second), + } +} diff --git a/urlscan.go b/urlscan.go new file mode 100644 index 0000000..af6e705 --- /dev/null +++ b/urlscan.go @@ -0,0 +1,52 @@ +package assetfinder + +import ( + "fmt" + "net/url" +) + +type URLScan struct{} + +type URLScanResults struct { + Results []struct { + Task struct { + URL string `json:"url"` + } `json:"task"` + + Page struct { + URL string `json:"url"` + } `json:"page"` + } `json:"results"` +} + +func (URLScan) FetchSubDomains(domain string) ([]string, error) { + resp := URLScanResults{} + err := fetchJSON( + fmt.Sprintf("https://urlscan.io/api/v1/search/?q=domain:%s", domain), + resp, + ) + if err != nil { + return []string{}, err + } + output := make([]string, 0) + + for _, r := range resp.Results { + u, err := url.Parse(r.Task.URL) + if err != nil { + continue + } + + output = append(output, u.Hostname()) + } + + for _, r := range resp.Results { + u, err := url.Parse(r.Page.URL) + if err != nil { + continue + } + + output = append(output, u.Hostname()) + } + + return output, nil +}