pijul/pijul - Change OGJFEWHUMFIZYBS456FPNUN3KEUDZXIJNFGPEMJGQDC7INZMHQZQC

Fixing missing dependencies on partial clones

This patch sends the list of patches + all their dependencies that touch a path, instead of letting the client download the dependencies after the initial download.

Created by pmeunier on March 4, 2024

OGJFEWHUMFIZYBS456FPNUN3KEUDZXIJNFGPEMJGQDC7INZMHQZQC

Dependencies

In channels

main

In discussions

#882 [CONSISTENCY ALARM] partial clone sometimes fails

Change contents

Replacement in pijul-remote/src/local.rs at line 71 [5.79745]
B:BD[5.80668] → [6.2125:2143]
```
        mut f: F,
```
[5.80668]
[6.2143]
```
        f: F,
```
Deletion in pijul-remote/src/local.rs at line 76 [5.79745]
B:BD[7.4367] → [8.1024:1126]
B:BD[8.1126] → [3.722:766]
∅:D[9.417] → [8.1170:1181]
∅:D[3.766] → [8.1170:1181]
B:BD[8.1170] → [8.1170:1181]
```
        let store = libpijul::changestore::filesystem::FileSystem::from_root(
            &self.root,
            pijul_repository::max_files()?,
        );
```

Insertion in pijul-remote/src/local.rs at line 86 [5.79745]

[5.81183]

[7.4408]

        self.download_changelist_(f, a, from, paths, &remote_txn, &remote_channel)
    }
    pub fn download_changelist_<
        A,
        T: libpijul::ChannelTxnT + libpijul::TxnTExt + libpijul::DepsTxnT + libpijul::GraphTxnT,
        F: FnMut(&mut A, u64, Hash, Merkle, bool) -> Result<(), anyhow::Error>,
    >(
        &mut self,
        mut f: F,
        a: &mut A,
        from: u64,
        paths: &[String],
        remote_txn: &T,
        remote_channel: &ChannelRef<T>,
    ) -> Result<HashSet<Position<Hash>>, anyhow::Error> {
        let store = libpijul::changestore::filesystem::FileSystem::from_root(
            &self.root,
            pijul_repository::max_files()?,
        );

Replacement in pijul-remote/src/local.rs at line 113 [5.79745]

∅:D[4.52] → [10.97:241]

B:BD[11.10438] → [10.97:241]

                    libpijul::fs::iter_graph_descendants(&remote_txn, &remote_channel.read(), p)?
                        .map(|x| x.unwrap()),

[4.52]

[4.53]

                    libpijul::fs::iter_graph_descendants(
                        remote_txn,
                        remote_txn.graph(&*remote_channel.read()),
                        p,
                    )?
                    .map(|x| x.unwrap()),

Replacement in pijul-remote/src/local.rs at line 138 [5.79745]

B:BD[12.21242] → [12.21242:21290]

∅:D[13.1033] → [11.10768:10802]

∅:D[14.1179] → [11.10768:10802]

∅:D[12.21290] → [11.10768:10802]

B:BD[11.10768] → [11.10768:10802]

B:BD[11.10802] → [15.8963:9223]

        for x in remote_txn.log(&*rem, from)? {
            let (n, (h, m)) = x?;
            assert!(n >= from);
            let h_int = remote_txn.get_internal(h)?.unwrap();
            if paths_.is_empty()
                || paths_.iter().any(|x| {
                    remote_txn
                        .get_touched_files(x, Some(h_int))

[12.21242]

[15.9223]

        if paths_.is_empty() {
            for x in remote_txn.log(&*rem, from)? {
                debug!("log {:?}", x);
                let (n, (h, m)) = x?;
                assert!(n >= from);
                debug!("put_remote {:?} {:?} {:?}", n, h, m);
                if tags.get(tagsi) == Some(&n) {
                    f(a, n, h.into(), m.into(), true)?;
                    tagsi += 1;
                } else {
                    f(a, n, h.into(), m.into(), false)?;
                }
            }
        } else {
            let mut hashes = HashMap::new();
            let mut stack = Vec::new();
            for x in remote_txn.log(&*rem, from)? {
                debug!("log {:?}", x);
                let (n, (h, m)) = x?;
                assert!(n >= from);
                let h_int = remote_txn.get_internal(h)?.unwrap();
                if paths_.is_empty()
                    || paths_.iter().any(|x| {
                        let y = remote_txn.get_touched_files(x, Some(h_int)).unwrap();
                        debug!("x {:?} {:?}", x, y);
                        y == Some(h_int)
                    })
                {
                    stack.push((*h_int, *m, n));
                }
            }
            while let Some((h_int, m, n)) = stack.pop() {
                if hashes.insert(h_int, (m, n)).is_some() {
                    continue;
                }
                for d in remote_txn.iter_dep(&h_int)? {
                    let (&h_int_, &d) = d?;
                    if h_int_ < h_int {
                        continue;
                    } else if h_int_ > h_int {
                        break;
                    }
                    let n = remote_txn
                        .get_changeset(remote_txn.changes(&*rem), &d)

Replacement in pijul-remote/src/local.rs at line 184 [5.79745]

B:BD[15.9257] → [15.9257:9325]

                        .is_some()
                })
            {

[15.9257]

[15.9325]

                        .unwrap();
                    let m = remote_txn
                        .get_revchangeset(remote_txn.rev_changes(&*rem), &n)
                        .unwrap()
                        .unwrap()
                        .b;
                    stack.push((d, m.into(), (*n).into()))
                }
            }
            let mut hashes: Vec<_> = hashes.into_iter().collect();
            hashes.sort_by_key(|(_, (_, n))| *n);
            for (h_int, (m, n)) in hashes {
                let h = remote_txn.get_external(&h_int)?.unwrap();

Insertion in pijul-remote/src/local.rs at line 287 [5.79745]

[16.1245]

[17.12157]

                libpijul::changestore::filesystem::pop_filename(&mut self.changes_dir);

Replacement in pijul-remote/src/local.rs at line 289 [5.79745]

B:BD[17.12233] → [18.1463:1509]

                send.send((c, false)).await?;

[17.12233]

[16.1246]

                send.send((c, true)).await?;

Deletion in pijul-remote/src/lib.rs at line 1348 [5.52474]
∅:D[19.478] → [20.1100:1156]
B:BD[20.1100] → [20.1100:1156]
```
        let mut change_path = repo.changes_dir.clone();
```

Deletion in pijul-remote/src/lib.rs at line 1359 [5.52474]

B:BD[20.1522] → [20.1522:1702]

                        libpijul::changestore::filesystem::push_filename(&mut change_path, &hash);
                        std::fs::create_dir_all(change_path.parent().unwrap())?;

Deletion in pijul-remote/src/lib.rs at line 1377 [5.52474]

B:BD[20.2570] → [20.2570:2661]

                        libpijul::changestore::filesystem::pop_filename(&mut change_path);

Insertion in pijul/src/commands/protocol.rs at line 5 [5.123281]
[21.1913]
[21.1913]
```
use std::sync::Arc;
```

Replacement in pijul/src/commands/protocol.rs at line 56 [5.123281]

B:BD[15.15837] → [14.11368:11418]

        let txn = repo.pristine.arc_txn_begin()?;

[15.15837]

[5.124930]

        let pristine = Arc::new(repo.pristine);
        let txn = pristine.arc_txn_begin()?;

Replacement in pijul/src/commands/protocol.rs at line 129 [5.123281]

B:BD[5.126705] → [7.11321:11419]

                let mut paths = HashSet::new();
                debug!("cap[3] = {:?}", &cap[3]);

[5.126705]

[14.11835]

                let mut paths = Vec::new();

Replacement in pijul/src/commands/protocol.rs at line 131 [5.123281]

∅:D[7.11419] → [5.126749:126816]

∅:D[14.11873] → [5.126749:126816]

B:BD[5.126749] → [5.126749:126816]

B:BD[5.126816] → [7.11420:11484]

∅:D[7.11484] → [5.126875:127036]

B:BD[5.126875] → [5.126875:127036]

B:BD[5.127036] → [11.15583:15635]

                for r in CHANGELIST_PATHS.captures_iter(&cap[3]) {
                    let s: String = r[1].replace("\\\"", "\"");
                    if let Ok((p, ambiguous)) = txn.follow_oldest_path(&repo.changes, &channel, &s)
                    {
                        if ambiguous {
                            bail!("Ambiguous path")

[14.11873]

[5.127231]

                {
                    for r in CHANGELIST_PATHS.captures_iter(&cap[3]) {
                        let s: String = r[1].replace("\\\"", "\"");
                        if let Ok((p, ambiguous)) =
                            txn.follow_oldest_path(&repo.changes, &channel, &s)
                        {
                            if ambiguous {
                                bail!("Ambiguous path")
                            }
                            let h: libpijul::Hash = txn.get_external(&p.change)?.unwrap().into();
                            writeln!(o, "{}.{}", h.to_base32(), p.pos.0)?;
                            paths.push(s);
                        } else {
                            debug!("protocol line: {:?}", buf);
                            bail!("Protocol error")

Deletion in pijul/src/commands/protocol.rs at line 147 [5.123281]

B:BD[5.127257] → [22.12298:12392]

∅:D[22.12392] → [7.11554:11666]

∅:D[11.15706] → [7.11554:11666]

B:BD[7.11554] → [7.11554:11666]

B:BD[7.11666] → [11.15707:15745]

B:BD[11.15745] → [10.242:335]

∅:D[10.335] → [11.15845:15926]

∅:D[14.11973] → [11.15845:15926]

∅:D[13.17936] → [11.15845:15926]

B:BD[11.15845] → [11.15845:15926]

∅:D[7.11881] → [5.127295:127324]

∅:D[11.15926] → [5.127295:127324]

B:BD[5.127295] → [5.127295:127324]

B:BD[5.127324] → [11.15927:16035]

                        let h: libpijul::Hash = txn.get_external(&p.change)?.unwrap().into();
                        writeln!(o, "{}.{}", h.to_base32(), p.pos.0)?;
                        paths.insert(p);
                        paths.extend(
                            libpijul::fs::iter_graph_descendants(&*txn, &channel.read(), p)?
                                .map(|x| x.unwrap()),
                        );
                    } else {
                        debug!("protocol line: {:?}", buf);
                        bail!("Protocol error")

Deletion in pijul/src/commands/protocol.rs at line 149 [5.123281]

B:BD[5.127543] → [7.11882:11929]

B:BD[7.11929] → [12.33763:33955]

                debug!("paths = {:?}", paths);
                let tags: Vec<u64> = txn
                    .iter_tags(txn.tags(&*channel.read()), from)?
                    .map(|k| (*k.unwrap().0).into())
                    .collect();

Replacement in pijul/src/commands/protocol.rs at line 150 [5.123281]

∅:D[12.33990] → [14.11974:12034]

B:BD[7.11929] → [14.11974:12034]

∅:D[14.12034] → [11.16097:16202]

∅:D[13.17998] → [11.16097:16202]

B:BD[11.16097] → [11.16097:16202]

∅:D[11.16202] → [5.127675:127715]

B:BD[5.127675] → [5.127675:127715]

B:BD[5.127715] → [7.11930:11980]

B:BD[7.11980] → [22.12393:12532]

∅:D[22.12532] → [7.12078:12105]

∅:D[11.16342] → [7.12078:12105]

B:BD[7.12078] → [7.12078:12105]

∅:D[7.12105] → [5.127871:127893]

B:BD[5.127871] → [5.127871:127893]

B:BD[5.127893] → [22.12533:12631]

B:BD[22.12631] → [12.33991:34068]

                for x in txn.log(&*channel.read(), from)? {
                    let (n, (h, m)) = x?;
                    let h_int = txn.get_internal(h)?.unwrap();
                    if paths.is_empty()
                        || paths.iter().any(|x| {
                            x.change == *h_int
                                || txn.get_touched_files(x, Some(h_int)).unwrap().is_some()
                        })
                    {
                        let h: Hash = h.into();
                        let m: Merkle = m.into();
                        if paths.is_empty() && tags.get(tagsi) == Some(&n) {

[12.33990]

[2.0]

                (pijul_remote::local::Local {
                    channel: (&cap[1]).to_string(),
                    root: PathBuf::new(),
                    changes_dir: PathBuf::new(),
                    pristine: pristine.clone(),
                    name: String::new(),
                })
                .download_changelist_(
                    |_, n, h, m, is_tag| {
                        if is_tag {

Replacement in pijul/src/commands/protocol.rs at line 165 [5.123281]

∅:D[23.3473] → [5.127975:128015]

B:BD[5.127975] → [5.127975:128015]

                    }
                }

[23.3473]

[24.1616]

                        Ok(())
                    },
                    &mut (),
                    from,
                    &paths,
                    &*txn,
                    &channel,
                )?;