ember/mediabias-plotter - Change B3H475WM3JE532SL7IGJIQBRXWHNDTHP2LH5IL67N46Z6QM75SFAC

first phase

Created by ember on January 10, 2021

B3H475WM3JE532SL7IGJIQBRXWHNDTHP2LH5IL67N46Z6QM75SFAC

Dependencies

In channels

main

Change contents

Insertion in src/main.rs at line 7 [3.10]
[2.94]
[2.94]
```
use bytes::Bytes;
use clap::{clap_app};
```
Deletion in src/main.rs at line 12 [3.10]
B:BD[3.48] → [2.147:173]
```
use serde_bytes::ByteBuf;
```
Insertion in src/main.rs at line 13 [3.10]
[2.188]
[3.48]
```
use tokio::{sync::broadcast};
```
Replacement in src/main.rs at line 40 [3.10]
B:BD[3.616] → [2.189:213]
```
    page_html: ByteBuf,
```
[3.616]
[3.653]
```
    page_html: Bytes,
```
Replacement in src/main.rs at line 43 [3.10]
B:BD[3.736] → [2.214:244]
```
    cleaned_summary: ByteBuf,
```
[3.736]
[3.779]
```
    cleaned_summary: Bytes,
```

Insertion in src/main.rs at line 46 [3.10]

[3.782]

[2.245]

#[derive(Serialize, Deserialize, Debug, PartialEq, PartialOrd, Hash, Eq)]

Replacement in src/main.rs at line 49 [3.10]
B:BD[2.288] → [2.288:307]
```
    data: ByteBuf,
```
[2.288]
[2.307]
```
    data: Bytes,
```

Replacement in src/main.rs at line 52 [3.10]

B:BD[2.310] → [2.310:461]

// map from original scraped url to retrieved page.
async fn all_urls_on_site_everywhere(cl: &Client) -> anyhow::Result<HashMap<Url, PageCandidate>> {

[2.310]

[2.461]

/// add elements to confirmed
async fn all_urls_on_site_everywhere(
    mut stop: broadcast::Receiver<()>,
    cl: &Client,
    confirmed: &mut HashMap<Url, PageCandidate>,
) -> anyhow::Result<()> {

Deletion in src/main.rs at line 60 [3.10]

B:BD[2.574] → [2.574:643]

    let mut confirmed: HashMap<Url, PageCandidate> = HashMap::new();

Replacement in src/main.rs at line 64 [3.10]

B:BD[2.827] → [2.827:916]

    while let Some(next_url) = pending.iter().next() {
        pending.remove(next_url);

[2.827]

[2.916]

    while let Some(next_url) = pending.iter().cloned().next() {
        pending.remove(&next_url);

Replacement in src/main.rs at line 67 [3.10]

B:BD[2.917] → [2.917:1165]

        required_sleep();
        let pg = cl.get(next_url).send().await?;
        confirmed.push(PageCandidate {
            final_url: pg.url.clone(),
            data: pg.bytes().await?,
        });
        let doc = Html::parse_document(data);

[2.917]

[2.1165]

        required_sleep().await;
        let pg = cl.get(next_url.clone()).send().await?;
        let final_url = pg.url().clone();
        let data = pg.bytes().await?;
        let doc = Html::parse_document(std::str::from_utf8(&data)?);

Replacement in src/main.rs at line 77 [3.10]

B:BD[2.1362] → [2.1362:1504]

                        if !confirmed.contains_key(potentially_new_url)
                            && !pending.contains(potentially_new_url)

[2.1362]

[2.1504]

                        if !confirmed.contains_key(&potentially_new_url)
                            && !pending.contains(&potentially_new_url)
                            && potentially_new_url.domain() == Some("mediabiasfactcheck.com")

Replacement in src/main.rs at line 81 [3.10]

B:BD[2.1530] → [2.1530:1593]

                            pending.push(potentially_new_url);

[2.1530]

[2.1593]

                            tracing::trace!("chosing to visit {}", potentially_new_url);
                            pending.insert(potentially_new_url);
                        } else {
                            tracing::trace!("skipping {}", potentially_new_url);

Insertion in src/main.rs at line 91 [3.10]

[2.1711]

        confirmed.insert(next_url.clone(), PageCandidate { final_url, data });
        tracing::info!("finished {}", next_url);
        if stop.try_recv().is_ok() {
            break;
        }

Replacement in src/main.rs at line 98 [3.10]
B:BD[2.1718] → [2.1718:1736]
```
    Ok(confirmed)
```
[2.1718]
[2.1736]
```
    Ok(())
```

Replacement in src/main.rs at line 103 [3.10]

∅:D[2.1794] → [3.794:825]

B:BD[3.794] → [3.794:825]

    println!("Hello, world!");

[2.1794]

[3.825]

    let app = clap_app!(myapp =>
        (version: "0.1")
        (author: "ember")
        (about: "create a plot of media bias according to mediabiasfactcheck.com")
        (@subcommand first_scrape =>
            (about: "crawl all the website's urls ")
            (@arg verbose: -v --verbose "Verbose logging")
        )
    );
    let args = app.get_matches();
    let (stop_tx, stop) = broadcast::channel(1);
    let interrupter = tokio::task::spawn(async move {
        tokio::signal::ctrl_c()
            .await
            .expect("couldn't wait for signal?");
        stop_tx.send(()).expect("failed to broadcast stop signal");
    });
    let mut cl = reqwest::Client::new();
    tracing_subscriber::fmt().init();
    if let Some(phase1) = args.subcommand_matches("first-scrape") {
        if phase1.is_present("verbose") {
            // TODO: figure out how to decrease log level unless this is  passed?
        }
        // load prev state from blob
        let mut confirmed = bincode::deserialize_from(std::io::BufReader::new(
            std::fs::File::open("phase1.blob").unwrap(),
        ))
        .unwrap_or(HashMap::new());
        all_urls_on_site_everywhere(stop, &mut cl, &mut confirmed).await?;
        bincode::serialize_into(
            std::io::BufWriter::new(std::fs::File::create("phase1.blob").expect("sad")),
            &confirmed,
        )?;
    }
    std::mem::forget(interrupter);
    Ok(())

Replacement in Cargo.toml at line 19 [4.20]

B:BD[2.1995] → [2.1995:2011]

scraper = "0.12"

[2.1995]

scraper = "0.12"
bytes = { version = "1", features = ["serde"]}
clap = "3.0.0-beta.2"
tracing = "0.1"
bincode = "1.3"
tracing-subscriber = "0.2.15"

Insertion in Cargo.lock at line 4 [4.450]

[4.554]

[2.2012]

name = "ansi_term"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
dependencies = [
 "winapi",
]
[[package]]

Insertion in Cargo.lock at line 37 [4.450]

[2.2480]

]
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
 "hermit-abi",
 "libc",
 "winapi",

Insertion in Cargo.lock at line 63 [4.450]

[4.1647]

[2.2598]

name = "bincode"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d"
dependencies = [
 "byteorder",
 "serde",
]
[[package]]

Insertion in Cargo.lock at line 110 [4.450]
[2.3207]
[4.2487]
```
dependencies = [
 "serde",
]
```

Insertion in Cargo.lock at line 147 [4.450]

[4.4140]

name = "clap"
version = "3.0.0-beta.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142"
dependencies = [
 "atty",
 "bitflags",
 "clap_derive",
 "indexmap",
 "lazy_static",
 "os_str_bytes",
 "strsim",
 "termcolor",
 "textwrap",
 "unicode-width",
 "vec_map",
]
[[package]]
name = "clap_derive"
version = "3.0.0-beta.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1"
dependencies = [
 "heck",
 "proc-macro-error",
 "proc-macro2",
 "quote",
 "syn",
]
[[package]]

Insertion in Cargo.lock at line 396 [4.450]

[4.12304]


[[package]]
name = "heck"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac"
dependencies = [
 "unicode-segmentation",
]

Insertion in Cargo.lock at line 616 [4.450]

[4.17243]

[2.6455]

name = "matchers"
version = "0.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1"
dependencies = [
 "regex-automata",
]
[[package]]

Insertion in Cargo.lock at line 636 [4.450]
[2.6598]
[4.17501]
```
 "bincode",
 "bytes",
```
Insertion in Cargo.lock at line 639 [4.450]
[4.17512]
[2.6599]
```
 "clap",
```
Insertion in Cargo.lock at line 646 [4.450]
[2.6645]
[2.6645]
```
 "tracing",
 "tracing-subscriber",
```

Insertion in Cargo.lock at line 794 [4.450]

[4.21788]

name = "os_str_bytes"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85"
[[package]]

Insertion in Cargo.lock at line 955 [4.450]

[2.9792]

name = "proc-macro-error"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [
 "proc-macro-error-attr",
 "proc-macro2",
 "quote",
 "syn",
 "version_check",
]
[[package]]
name = "proc-macro-error-attr"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
 "proc-macro2",
 "quote",
 "version_check",
]
[[package]]

Insertion in Cargo.lock at line 1060 [4.450]

[4.29250]

[4.29672]

name = "regex"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
dependencies = [
 "regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4"
dependencies = [
 "byteorder",
 "regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
[[package]]

Insertion in Cargo.lock at line 1279 [4.450]

[4.34851]

[2.11118]

]
[[package]]
name = "sharded-slab"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3"
dependencies = [
 "lazy_static",

Insertion in Cargo.lock at line 1360 [4.450]

[4.37007]

name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]

Insertion in Cargo.lock at line 1402 [4.450]

[4.38066]

name = "termcolor"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
dependencies = [
 "winapi-util",
]
[[package]]
name = "textwrap"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789"
dependencies = [
 "unicode-width",
]
[[package]]

Insertion in Cargo.lock at line 1426 [4.450]

[4.38260]

name = "thread_local"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447"
dependencies = [
 "lazy_static",
]
[[package]]

Insertion in Cargo.lock at line 1541 [4.450]
[2.13119]
[2.13119]
```
 "tracing-attributes",
```

Insertion in Cargo.lock at line 1543 [4.450]

[2.13136]

[4.40913]

]
[[package]]
name = "tracing-attributes"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada"
dependencies = [
 "proc-macro2",
 "quote",
 "syn",

Insertion in Cargo.lock at line 1576 [4.450]

[4.41843]

name = "tracing-log"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9"
dependencies = [
 "lazy_static",
 "log",
 "tracing-core",
]
[[package]]
name = "tracing-serde"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b"
dependencies = [
 "serde",
 "tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401"
dependencies = [
 "ansi_term",
 "chrono",
 "lazy_static",
 "matchers",
 "regex",
 "serde",
 "serde_json",
 "sharded-slab",
 "smallvec",
 "thread_local",
 "tracing",
 "tracing-core",
 "tracing-log",
 "tracing-serde",
]
[[package]]

Insertion in Cargo.lock at line 1643 [4.450]

[4.42956]

name = "unicode-segmentation"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]

Insertion in Cargo.lock at line 1686 [4.450]

[4.44696]

name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version_check"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
[[package]]

Insertion in Cargo.lock at line 1812 [4.450]

[4.46215]


[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
 "winapi",
]