June 05 2026

I wrote my first basic web scraping function which takes a url string, fetches it, finds the title and returns the title string.

func Scrape(url string) string {
	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
		log.Fatal("Error: this url doesnt start with http(s)")
	}
 
	res, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
	}
	
	// THE ACTUAL HTML SCRAPING
	// Load the HTML document
	doc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		log.Fatal(err)
	}
 
	// Find the title
	title := doc.Find("title").First().Text()
 
	return title
 
}

As you can see it is very easy and simple to fetch the title, but I’ll have to extend it for it to find all anchor tags and not their text but their href attribute.

Let’s modify the code as follows:

func Scrape(url string) (string, []string) {
	
	// ... previous code ...
 
	// Find the title
	title := doc.Find("title").First().Text()
 
	// Find urls
	var urls []string
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		url, exists := s.Attr("href")
		if exists {
			urls = append(urls, url)
		}
	})
 
	return title, urls
 
}

Now you can access the title and urls in an easy way:

title, urls := Scrape('https://wannesg.be')
fmt.Println(title)
for i := 0; i < len(urls); i++ {
	fmt.Println(urls[i])
}

‍I built a simple main function using basic FIFO queuing where every new found url is added to a queue and the oldest item is processed next. It starts with a seed url and then

1. filters out non-absolute links
2. checks if it is already processed
3. scrapes it
4. adds all new found links that are not processed and not queued already to the queue
5. removes processed url from queue and add it to finished url

it uses goroutines for concurrent scraping so it goes a little faster.

var stop = false
const seed = "https://vimexx.be"
var urls = []string{}
var doneUrls = []string{seed} // add seed to queue
 
func main() {
 
	_, urls = Scrape(seed)
 
	for !stop {
		// go through all queued urls
		for i := range urls {
			url := urls[i]
 
			// if the url is absolute and has not been processed before, continue
			// todo: convert relative links to absolute and process them too
			if (strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) && !slices.Contains(doneUrls, url) {
				// scrape it
				go func() {
					_, newUrls := Scrape(urls[i])
 
					// go through all newly found urls
					for j := range newUrls {
 
						// if its already been processed or queued, skip it
						if !slices.Contains(doneUrls, newUrls[j]) && !slices.Contains(urls, newUrls[j]) {
							urls = append(urls, newUrls[j])
						}
					}
				}()
 
				// remove processed url from queue and add to doneurls
				urls = removeFromSlice(urls, i)
				doneUrls = append(doneUrls, url)
				sleep(200)
			}
		}
	}
 
}