use clap::{clap_app};

use clap::clap_app;

use tokio::{sync::broadcast};

use tokio::sync::broadcast;
use tracing::{error, info, metadata::LevelFilter, Level};
use tracing_subscriber::EnvFilter;

fn find_new_links(
    data: &str,
    old_links: &HashMap<Url, PageCandidate>,
    pending: &mut HashSet<Url>,
) -> anyhow::Result<()> {
    let afinder = Selector::parse("a").map_err(|e| anyhow!("this is no good :( {:?}", e))?;

    let doc = Html::parse_document(data);
    for a_elem in doc.select(&afinder) {
        if let Some(link) = a_elem.value().attr("href") {
            match Url::parse(link) {
                Ok(potentially_new_url) => {
                    if !old_links.contains_key(&potentially_new_url)
                        && !pending.contains(&potentially_new_url)
                        && potentially_new_url.domain() == Some("mediabiasfactcheck.com")
                    {
                        println!("chosing to visit {}", potentially_new_url);
                        pending.insert(potentially_new_url);
                    } else {
                        tracing::trace!("skipping {}", potentially_new_url);
                    }
                }
                _ => {}
            }
        }
    }
    Ok(())
}

    let mut pending: HashSet<Url> =
        iter::once("https://mediabiasfactcheck.com/".try_into()?).collect();

    let mut pending: HashSet<Url> = if confirmed.is_empty() {
        info!("starting from root, with no cache");
        iter::once("https://mediabiasfactcheck.com/".try_into()?).collect()
    } else {
        let mut new = HashSet::new();
        confirmed
            .values()
            .map(|v| {
                find_new_links(
                    std::str::from_utf8(&v.data).expect("nonutf8"),
                    &confirmed,
                    &mut new,
                )
            })
            .for_each(|_| ());
        info!("scraping through {:?}", new);
        new
    };

    let required_sleep = || tokio::time::sleep(std::time::Duration::from_secs_f32(0.25));
    let afinder = Selector::parse("a").map_err(|e| anyhow!("this is no good :( {:?}", e))?;

    let required_sleep = || tokio::time::sleep(std::time::Duration::from_secs_f32(0.05));

        let doc = Html::parse_document(std::str::from_utf8(&data)?);
        for a_elem in doc.select(&afinder) {
            if let Some(link) = a_elem.value().attr("href") {
                match Url::parse(link) {
                    Ok(potentially_new_url) => {
                        if !confirmed.contains_key(&potentially_new_url)
                            && !pending.contains(&potentially_new_url)
                            && potentially_new_url.domain() == Some("mediabiasfactcheck.com")
                        {
                            tracing::trace!("chosing to visit {}", potentially_new_url);
                            pending.insert(potentially_new_url);
                        } else {
                            tracing::trace!("skipping {}", potentially_new_url);
                        }
                    }
                    _ => {}
                }
            }
        }

        tracing::info!("finished {}", next_url);

        println!("finished {}", next_url);

    tracing_subscriber::fmt()
        .with_max_level(Level::WARN)
        .with_env_filter(
            EnvFilter::from_default_env()
                .add_directive(LevelFilter::from_level(Level::WARN).into())
                .add_directive("mediabias-parsing=trace".parse()?),
        )
        .init();

    tracing_subscriber::fmt().init();

    if let Some(phase1) = args.subcommand_matches("first-scrape") {

    if let Some(phase1) = args.subcommand_matches("first_scrape") {

        let mut confirmed = bincode::deserialize_from(std::io::BufReader::new(
            std::fs::File::open("phase1.blob").unwrap(),
        ))

        let mut confirmed = {
            let f = std::fs::File::open("phase1.blob").map_err(|e| anyhow!(e));
            f.and_then(|f| {
                bincode::deserialize_from(std::io::BufReader::new(f)).map_err(|e| anyhow!(e))
            })
        }

        all_urls_on_site_everywhere(stop, &mut cl, &mut confirmed).await?;

        let r = all_urls_on_site_everywhere(stop, &mut cl, &mut confirmed).await;
        match r {
            Err(e) => error!("big sad: {}", e),
            Ok(_) => {}
        }