chore(examples): add glob example

examples/Cargo.toml

@@ -101,4 +101,14 @@ required-features = ["spider/sync", "spider/encoding"]
 [[example]]
 name = "advanced_configuration"
 path = "advanced_configuration.rs"
-required-features = ["spider/sync"]
+required-features = ["spider/sync"]
+
+[[example]]
+name = "url_glob"
+path = "url_glob.rs"
+required-features = ["spider/sync", "spider/glob"]
+
+[[example]]
+name = "url_glob_subdomains"
+path = "url_glob_subdomains.rs"
+required-features = ["spider/sync", "spider/glob"]

examples/README.md

@@ -56,7 +56,14 @@ Use different encodings for the page. [Encoding](./encoding.rs).
 
 - `cargo run --example encoding --features encoding`
 
-
 Use advanced configuration re-use. [Advanced Configuration](./advanced_configuration.rs).
 
-- `cargo run --example advanced_configuration`
+- `cargo run --example advanced_configuration`
+
+Use URL globbing for a domain. [URL Globbing](./glob.rs).
+
+- `cargo run --example glob --features glob`
+
+Use URL globbing for a domain and subdomains. [URL Globbing Subdomains](./url_glob_subdomains.rs).
+
+- `cargo run --example url_glob_subdomains --features glob`

examples/advanced_configuration.rs

@@ -1,8 +1,8 @@
 //! `cargo run --example advanced_configuration`
 extern crate spider;
 
-use spider::{tokio, website::Website, configuration::Configuration};
-use std::{time::Instant, io::Error};
+use spider::{configuration::Configuration, tokio, website::Website};
+use std::{io::Error, time::Instant};
 
 const CAPACITY: usize = 5;
 const CRAWL_LIST: [&str; CAPACITY] = [

examples/url_glob.rs

@@ -0,0 +1,34 @@
+//! `cargo run --example url_glob --features glob`
+extern crate spider;
+
+use spider::tokio;
+use spider::website::Website;
+use std::time::Instant;
+
+#[tokio::main]
+async fn main() {
+    let mut website: Website = Website::new(
+        "https://rsseau.fr/blog/{lazy-load-components,gnu-parallel,migrate-from-jekyll-to-gatsby}",
+    );
+    website
+        .configuration
+        .blacklist_url
+        .insert(Default::default())
+        .push("https://rsseau.fr/resume".into());
+
+    let start = Instant::now();
+    website.crawl().await;
+    let duration = start.elapsed();
+
+    let links = website.get_links();
+
+    for link in links {
+        println!("- {:?}", link.as_ref());
+    }
+
+    println!(
+        "Time elapsed in website.crawl() is: {:?} for total pages: {:?}",
+        duration,
+        links.len()
+    )
+}

examples/url_glob_subdomains.rs

@@ -0,0 +1,30 @@
+//! `cargo run --example url_glob_subdomains --features glob`
+extern crate spider;
+
+use spider::tokio;
+use spider::website::Website;
+use std::time::Instant;
+
+#[tokio::main]
+async fn main() {
+    let mut website: Website = Website::new("https://{www,docs}.a11ywatch.com")
+        .with_subdomains(true)
+        .build()
+        .unwrap();
+
+    let start = Instant::now();
+    website.crawl().await;
+    let duration = start.elapsed();
+
+    let links = website.get_links();
+
+    for link in links {
+        println!("- {:?}", link.as_ref());
+    }
+
+    println!(
+        "Time elapsed in website.crawl() is: {:?} for total pages: {:?}",
+        duration,
+        links.len()
+    )
+}