grahamc/hydra2 - Change EPWEMRI23UN6C777AL2GWEOXYZRTPEPFC2SZVZ4JTWUGYVFDL5XAC

Allow determinism checking for entire jobsets

Setting

xxx-jobset-repeats = patchelf:master:2

will cause Hydra to perform every build step in the specified jobset 2 additional times (i.e. 3 times in total). Non-determinism is not fatal unless the derivation has the attribute "isDeterministic = true"; we just note the lack of determinism in the Hydra database. This will allow us to get stats about the (lack of) reproducibility of all of Nixpkgs.

Created by Eelco Dolstra on December 7, 2016

EPWEMRI23UN6C777AL2GWEOXYZRTPEPFC2SZVZ4JTWUGYVFDL5XAC

Dependencies

In channels

main

Change contents

Replacement in src/hydra-queue-runner/build-remote.cc at line 119 [6.85]

B:BD[5.215] → [5.215:274]

    unsigned int maxSilentTime, unsigned int buildTimeout,

[5.215]

[3.0]

    unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,

Replacement in src/hydra-queue-runner/build-remote.cc at line 266 [6.85]

B:BD[7.3765] → [4.261:427]

        if (GET_PROTOCOL_MINOR(remoteVersion) >= 3)
            // FIXME: make the number of repeats configurable.
            to << (step->isDeterministic ? 1 : 0);

[7.3765]

        if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
            to << repeats // == build-repeat
               << step->isDeterministic; // == enforce-determinism
        }

Insertion in src/hydra-queue-runner/build-remote.cc at line 299 [6.85]

[7.4660]

            if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
                result.timesBuilt = readInt(from);
                result.isNonDeterministic = readInt(from);
            }

Insertion in src/hydra-queue-runner/builder.cc at line 89 [9.133]
[8.553]
[9.2164]
```
    unsigned int repeats = step->isDeterministic ? 1 : 0;
```

Insertion in src/hydra-queue-runner/builder.cc at line 116 [9.133]

[10.90]

[11.401]

            }
            {
                auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName));
                if (i != jobsetRepeats.end())
                    repeats = std::max(repeats, i->second);

Replacement in src/hydra-queue-runner/builder.cc at line 130 [9.133]

∅:D[8.741] → [9.3215:3327]

B:BD[9.3215] → [9.3215:3327]

B:BD[9.3327] → [8.742:827]

        printMsg(lvlInfo, format("performing step ‘%1%’ on ‘%2%’ (needed by build %3% and %4% others)")
            % step->drvPath % machine->sshName % buildId % (dependents.size() - 1));

[8.741]

[9.3414]

        printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
            step->drvPath, repeats + 1, machine->sshName, buildId, (dependents.size() - 1));

Replacement in src/hydra-queue-runner/builder.cc at line 171 [9.133]

B:BD[9.4376] → [3.1061:1161]

            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, result, activeStep);

[9.4376]

[12.2415]

            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep);

Replacement in src/hydra-queue-runner/builder.cc at line 233 [9.133]

B:BD[9.5536] → [8.1885:1979]

∅:D[8.1979] → [13.0:79]

B:BD[14.530] → [13.0:79]

            finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, buildId,
                stepNr, machine->sshName, result.stepStatus, result.errorMsg);

[9.5536]

[9.5687]

            finishBuildStep(txn, result, buildId, stepNr, machine->sshName);

Replacement in src/hydra-queue-runner/builder.cc at line 287 [9.133]

B:BD[9.7356] → [14.531:620]

B:BD[14.620] → [8.1980:2047]

                finishBuildStep(txn, result.startTime, result.stopTime, result.overhead,
                    buildId, stepNr, machine->sshName, bsSuccess);

[9.7356]

[9.7478]

                finishBuildStep(txn, result, buildId, stepNr, machine->sshName);

Replacement in src/hydra-queue-runner/builder.cc at line 393 [9.133]

∅:D[15.477] → [14.691:784]

∅:D[12.3420] → [14.691:784]

B:BD[9.12039] → [14.691:784]

B:BD[14.784] → [8.2409:2505]

                    finishBuildStep(txn, result.startTime, result.stopTime, result.overhead,
                        buildId, stepNr, machine->sshName, result.stepStatus, result.errorMsg);

[15.477]

[15.478]

                    finishBuildStep(txn, result, buildId, stepNr, machine->sshName);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 267 [16.4840]

B:BD[16.9262] → [14.881:985]

B:BD[14.985] → [10.703:794]

∅:D[17.777] → [14.1071:1129]

∅:D[10.794] → [14.1071:1129]

B:BD[14.1071] → [14.1071:1129]

void State::finishBuildStep(pqxx::work & txn, time_t startTime, time_t stopTime, unsigned int overhead,
    BuildID buildId, unsigned int stepNr, const std::string & machine, BuildStatus status,
    const std::string & errorMsg, BuildID propagatedFrom)

[16.9262]

[16.9436]

void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
    BuildID buildId, unsigned int stepNr, const std::string & machine)

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 270 [16.4840]
B:BD[16.9438] → [6.9555:9600]
```
    assert(startTime);
    assert(stopTime);
```
[16.9438]
[18.391]
```
    assert(result.startTime);
    assert(result.stopTime);
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 273 [16.4840]

B:BD[18.413] → [14.1130:1315]

∅:D[19.942] → [18.414:535]

∅:D[14.1315] → [18.414:535]

∅:D[6.9757] → [18.414:535]

B:BD[16.9611] → [18.414:535]

B:BD[18.535] → [19.943:973]

        ("update BuildSteps set busy = 0, status = $1, propagatedFrom = $4, errorMsg = $5, startTime = $6, stopTime = $7, machine = $8, overhead = $9 where build = $2 and stepnr = $3")
        ((int) status)(buildId)(stepNr)
        (propagatedFrom, propagatedFrom != 0)
        (errorMsg, errorMsg != "")
        (startTime)(stopTime)

[18.413]

[14.1316]

        ("update BuildSteps set busy = 0, status = $1, errorMsg = $4, startTime = $5, stopTime = $6, machine = $7, overhead = $8, timesBuilt = $9, isNonDeterministic = $10 where build = $2 and stepnr = $3")
        ((int) result.stepStatus)(buildId)(stepNr)
        (result.errorMsg, result.errorMsg != "")
        (result.startTime)(result.stopTime)

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 278 [16.4840]

B:BD[14.1349] → [14.1349:1391]

        (overhead, overhead != 0).exec();

[14.1349]

[16.13421]

        (result.overhead, result.overhead != 0)
        (result.timesBuilt, result.timesBuilt > 0)
        (result.isNonDeterministic, result.timesBuilt > 1)
        .exec();

Insertion in src/hydra-queue-runner/hydra-queue-runner.cc at line 813 [16.4840]

[20.1250]

[21.3753]

    // FIXME: hacky mechanism for configuring determinism checks.
    for (auto & s : tokenizeString<Strings>(hydraConfig["xxx-jobset-repeats"])) {
        auto s2 = tokenizeString<std::vector<std::string>>(s, ":");
        if (s2.size() != 3) throw Error("bad value in xxx-jobset-repeats");
        jobsetRepeats.emplace(std::make_pair(s2[0], s2[1]), std::stoi(s2[2]));
    }

Insertion in src/hydra-queue-runner/state.hh at line 50 [22.1128]
[12.4209]
[12.4209]
```
    unsigned int timesBuilt = 0;
    bool isNonDeterministic = false;
```

Insertion in src/hydra-queue-runner/state.hh at line 419 [22.1128]

[23.1950]


    /* How often the build steps of a jobset should be repeated in
       order to detect non-determinism. */
    std::map<std::pair<std::string, std::string>, unsigned int> jobsetRepeats;

Replacement in src/hydra-queue-runner/state.hh at line 447 [22.1128]

B:BD[22.7255] → [14.1424:1502]

B:BD[14.1502] → [10.3425:3494]

∅:D[10.3494] → [17.1122:1214]

B:BD[14.1562] → [17.1122:1214]

∅:D[24.763] → [22.7453:7490]

∅:D[17.1214] → [22.7453:7490]

B:BD[22.7453] → [22.7453:7490]

    void finishBuildStep(pqxx::work & txn, time_t startTime, time_t stopTime,
        unsigned int overhead, BuildID buildId, unsigned int stepNr,
        const std::string & machine, BuildStatus status, const std::string & errorMsg = "",
        BuildID propagatedFrom = 0);

[22.7255]

[25.3136]

    void finishBuildStep(pqxx::work & txn, const RemoteResult & result, BuildID buildId, unsigned int stepNr,
        const std::string & machine);

Insertion in src/hydra-queue-runner/state.hh at line 500 [22.1128]
[5.1984]
[3.2341]
```
        unsigned int repeats,
```

Insertion in src/lib/Hydra/Schema/BuildSteps.pm at line 104 [27.1675]

[14.1621]

[26.722]

  is_nullable: 1
=head2 timesbuilt
  data_type: 'integer'
  is_nullable: 1
=head2 isnondeterministic
  data_type: 'boolean'

Insertion in src/lib/Hydra/Schema/BuildSteps.pm at line 145 [27.1675]

[14.1684]

[27.2685]

  "timesbuilt",
  { data_type => "integer", is_nullable => 1 },
  "isnondeterministic",
  { data_type => "boolean", is_nullable => 1 },

Replacement in src/lib/Hydra/Schema/BuildSteps.pm at line 218 [27.1675]

B:BD[28.499] → [14.1685:1827]

# Created by DBIx::Class::Schema::Loader v0.07043 @ 2016-02-16 18:04:52
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TRALbEoaF/OIOyERYCyxkw

[28.499]

[29.10118]

# Created by DBIx::Class::Schema::Loader v0.07045 @ 2016-12-07 13:48:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3FYkqSUfgWmiqZzmX8J4TA

Replacement in src/root/build.tt at line 53 [32.7350]
∅:D[30.69] → [31.985:1002]
∅:D[33.200] → [31.985:1002]
∅:D[34.240] → [31.985:1002]
∅:D[35.240] → [31.985:1002]
∅:D[36.958] → [31.985:1002]
∅:D[8.5497] → [31.985:1002]
B:BD[31.985] → [31.985:1002]
```
            <td>
```
[8.5497]
[31.1002]
```
            <td class="step-status">
```

Replacement in src/root/build.tt at line 57 [32.7350]

B:BD[31.1125] → [31.1125:1151]

                Succeeded

[31.1125]

[17.1215]

                [% IF step.isnondeterministic %]
                  <span class="warn">Succeeded with non-determistic result</span>
                [% ELSE %]
                  Succeeded
                [% END %]
                [% IF step.timesbuilt > 1 %]
                  ([% step.timesbuilt %] times)
                [% END %]

Replacement in src/root/build.tt at line 66 [32.7350]

∅:D[17.1258] → [37.6782:6911]

B:BD[31.1194] → [37.6782:6911]

                <span class="error"><strong>Aborted</strong>[% IF step.errormsg %]: [% HTML.escape(step.errormsg); END %]</span>

[17.1258]

[8.5498]

                <span class="error">Aborted</span>[% IF step.errormsg %]: <em>[% HTML.escape(step.errormsg) %]</em>[% END %]

Replacement in src/root/build.tt at line 80 [32.7350]

B:BD[4.836] → [4.836:903]

                <span class="error">Non-deterministic build</span>

[4.836]

[16.20001]

                <span class="error">Non-determinism detected</span> [% IF step.timesbuilt %] after [% step.timesbuilt %] times[% END %]

Replacement in src/root/build.tt at line 82 [32.7350]

B:BD[16.20041] → [16.20041:20125]

                <span class="error">Failed: [% HTML.escape(step.errormsg) %]</span>

[16.20041]

[31.1245]

                <span class="error">Failed</span>: <em>[% HTML.escape(step.errormsg) %]</em>

Replacement in src/root/build.tt at line 147 [32.7350]

B:BD[31.2323] → [31.2323:2391]

          [% INCLUDE renderBuildStatusIcon size=128, build=build %]

[31.2323]

[31.2391]

          [% INCLUDE renderBuildStatusIcon size=128 build=build %]

Insertion in src/root/static/css/hydra.css at line 134 [32.8691]
[2.230]
[38.1098]
```
}
td.step-status span.error {
    color: red;
    font-weight: bold;
```

Insertion in src/root/static/css/hydra.css at line 140 [32.8691]

[38.1100]


td.step-status span.warn {
    color: #aaaa00;
    font-weight: bold;
}

Insertion in src/sql/hydra.sql at line 291 [39.1]

[14.1927]


    -- How many times this build step was done (for checking determinism).
    timesBuilt    integer,

Insertion in src/sql/hydra.sql at line 295 [39.1]

[14.1928]

[40.5891]

    -- Whether this build step produced different results when repeated.
    isNonDeterministic boolean,

File addition: upgrade-52.sql (----------)

[41.3004]

alter table BuildSteps
  add column timesBuilt integer,
  add column isNonDeterministic boolean;