Add tests for automated .gz decompression

This is not about Content-Encoding: gzip, which is undone
automatically by Go net/http.

This is about files that are compressed as is, e.g. /sitemap.xml.gz,
without Content-Encoding: gzip being sent.

colly_test.go

@@ -17,6 +17,7 @@ package colly
 import (
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
@@ -43,6 +44,13 @@ Disallow: /disallowed
 Disallow: /allowed*q=
 `
 
+const testXML = `<?xml version="1.0" encoding="UTF-8"?>
+<page>
+	<title>Test Page</title>
+	<paragraph type="description">This is a test page</paragraph>
+	<paragraph type="description">This is a test paragraph</paragraph>
+</page>`
+
 func newUnstartedTestServer() *httptest.Server {
 	mux := http.NewServeMux()
 
@@ -69,13 +77,13 @@ func newUnstartedTestServer() *httptest.Server {
 
 	mux.HandleFunc("/xml", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/xml")
-		w.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?>
-<page>
-	<title>Test Page</title>
-	<paragraph type="description">This is a test page</paragraph>
-	<paragraph type="description">This is a test paragraph</paragraph>
-</page>
-		`))
+		w.Write([]byte(testXML))
+	})
+
+	mux.HandleFunc("/test.xml.gz", func(w http.ResponseWriter, r *http.Request) {
+		ww := gzip.NewWriter(w)
+		defer ww.Close()
+		ww.Write([]byte(testXML))
 	})
 
 	mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
@@ -1417,7 +1425,7 @@ func TestCollectorOnXMLWithHtml(t *testing.T) {
 	}
 }
 
-func TestCollectorOnXMLWithXML(t *testing.T) {
+func testCollectorOnXMLWithXML(t *testing.T, path string) {
 	ts := newTestServer()
 	defer ts.Close()
 
@@ -1450,7 +1458,7 @@ func TestCollectorOnXMLWithXML(t *testing.T) {
 		}
 	})
 
-	c.Visit(ts.URL + "/xml")
+	c.Visit(ts.URL + path)
 
 	if !titleCallbackCalled {
 		t.Error("Failed to call OnXML callback for <title> tag")
@@ -1461,6 +1469,14 @@ func TestCollectorOnXMLWithXML(t *testing.T) {
 	}
 }
 
+func TestCollectorOnXMLWithXML(t *testing.T) {
+	testCollectorOnXMLWithXML(t, "/xml")
+}
+
+func TestCollectorOnXMLWithXMLCompressed(t *testing.T) {
+	testCollectorOnXMLWithXML(t, "/test.xml.gz")
+}
+
 func TestCollectorVisitWithTrace(t *testing.T) {
 	ts := newTestServer()
 	defer ts.Close()