{
  "part": 8,
  "title": "XZ stream compression (transport.compression)",
  "reset": "standard",
  "steps": [
    { "id": "8.1", "title": "Enable compression and confirm round-trip",
      "config": { "mutations": [ { "op": "set", "path": "transport.compression.method", "value": "xz" } ], "restore": "none" },
      "actions": [
        { "do": "write_repeated_line", "path": "/storage/testing/src/ds1/big.txt",
          "line": "compressible line for testing xz output shrinkage abcdefgh", "count": 50000 },
        { "do": "zfs_snapshot", "dataset": "{SRC}/ds1@part8_1" },
        { "do": "run_sneakernet", "servername": "test-source", "verbosity": 1, "log": "part8_1", "expect_exit": 0,
          "capture": { "name": "LOG81" } },
        { "do": "list_matching", "dir": "{TRANSPORT}/datasets", "pattern": "^ds1\\.xz$",
          "capture": { "name": "XZ_FILES", "reduce": "count" } },
        { "do": "list_matching", "dir": "{TRANSPORT}/datasets", "pattern": "^ds1\\.xz\\.IV$",
          "capture": { "name": "XZ_IV_FILES", "reduce": "count" } },
        { "do": "run_sneakernet", "servername": "test-target", "verbosity": 1, "expect_exit": 0 }
      ],
      "assertions": [
        { "type": "regex_present", "var": "LOG81", "pattern": "(?s)xz -zc.*openssl enc", "message": "log shows xz appearing in the pipeline before openssl" },
        { "type": "count_equals", "var": "XZ_FILES", "n": 1, "message": "ds1's .xz file was written to datasets/ (anchored to ds1 specifically - ds2 is enabled too from Part 5 onward and produces its own ds2.xz here, which is correct and not this assertion's concern)" },
        { "type": "count_equals", "var": "XZ_IV_FILES", "n": 1, "message": "its .xz.IV sidecar was written" },
        { "type": "files_match", "a": "/storage/testing/src/ds1/big.txt", "b": "/storage/testing/dst/ds1/big.txt",
          "message": "PASS: contents match - the target transparently decompresses" }
      ]
    },
    { "id": "8.2", "title": "Verification still works under compression", "depends_on": ["8.1"],
      "_comment": "verifyStream: header from Part 4's restore and compression: xz from Step 8.1 are both still in effect (no config mutation needed here).",
      "actions": [
        { "do": "zfs_snapshot", "dataset": "{SRC}/ds1@part8_2" },
        { "do": "run_sneakernet", "servername": "test-source", "verbosity": 3, "log": "part8_2", "expect_exit": 0,
          "capture": { "name": "LOG82" } },
        { "do": "run_sneakernet", "servername": "test-target", "verbosity": 1, "expect_exit": 0 }
      ],
      "assertions": [
        { "type": "regex_present", "var": "LOG82", "pattern": "verifyTransportFile", "message": "header check still passes for the .xz file (decrypts and decompresses before checking ZFS magic bytes)" }
      ]
    },
    { "id": "8.3a", "title": "Compression actually shrinks the output - uncompressed baseline", "depends_on": ["8.2"],
      "config": { "mutations": [ { "op": "set", "path": "transport.compression.method", "value": "off", "style": "single" } ], "restore": "none" },
      "actions": [
        { "do": "zfs_destroy", "dataset": "{DST}/ds1", "recursive": 1, "missing_ok": 1 },
        { "do": "rm_matching", "dir": "{SN}", "pattern": "^sneakernet_target\\.status(\\.[0-9_.-]+)?$" },
        { "do": "empty_dir", "path": "{TRANSPORT}" },
        { "do": "zfs_snapshot", "dataset": "{SRC}/ds1@part8_3a" },
        { "do": "run_sneakernet", "servername": "test-source", "verbosity": 1, "expect_exit": 0 },
        { "do": "dir_size_bytes", "path": "{TRANSPORT}/datasets", "store_as": "UNCOMPRESSED" },
        { "do": "run_sneakernet", "servername": "test-target", "verbosity": 1, "expect_exit": 0 }
      ],
      "assertions": [
        { "type": "true", "var": "UNCOMPRESSED", "message": "uncompressed transport size was captured" }
      ]
    },
    { "id": "8.3b", "title": "Compression actually shrinks the output - compressed comparison", "depends_on": ["8.3a"],
      "config": { "mutations": [ { "op": "set", "path": "transport.compression.method", "value": "xz" } ], "restore": "none" },
      "actions": [
        { "do": "empty_dir", "path": "{TRANSPORT}" },
        { "do": "zfs_snapshot", "dataset": "{SRC}/ds1@part8_3b" },
        { "do": "run_sneakernet", "servername": "test-source", "verbosity": 1, "expect_exit": 0 },
        { "do": "dir_size_bytes", "path": "{TRANSPORT}/datasets", "store_as": "COMPRESSED" },
        { "do": "run_sneakernet", "servername": "test-target", "verbosity": 1, "expect_exit": 0 }
      ],
      "assertions": [
        { "type": "values_less_than", "a": "COMPRESSED", "b": "UNCOMPRESSED", "message": "compressed transport size is meaningfully smaller than uncompressed - big.txt's highly repetitive content should shrink a lot, not marginally" }
      ]
    },
    { "id": "8.4", "title": "threads / nice are honored in the pipeline", "depends_on": ["8.3b"],
      "config": { "mutations": [
        { "op": "set", "path": "transport.compression.threads", "value": "half" },
        { "op": "set", "path": "transport.compression.nice", "value": "10" }
      ], "restore": "none" },
      "actions": [
        { "do": "empty_dir", "path": "{TRANSPORT}" },
        { "do": "zfs_snapshot", "dataset": "{SRC}/ds1@part8_4" },
        { "do": "run_sneakernet", "servername": "test-source", "verbosity": 3, "log": "part8_4", "expect_exit": 0,
          "capture": { "name": "LOG84" } },
        { "do": "run_sneakernet", "servername": "test-target", "verbosity": 1, "expect_exit": 0 }
      ],
      "assertions": [
        { "type": "regex_present", "var": "LOG84", "pattern": "nice -n 10 xz -zc -6 -T[0-9]+", "message": "compression pipeline shows nice and a resolved thread count" }
      ]
    },
    { "id": "8.4-cleanup", "title": "Restore threads: 0 and nice: 0", "depends_on": ["8.4"],
      "config": { "restore": "none", "mutations": [
        { "op": "set", "path": "transport.compression.threads", "value": "0" },
        { "op": "set", "path": "transport.compression.nice", "value": "0" }
      ] }
    },
    { "id": "8.5", "title": "(Optional) Mixed drive: one compressed file, one plain file, same trip", "classification": "record_only",
      "depends_on": ["8.4-cleanup"],
      "_comment": "classification=record_only (not 'optional'): per TESTING_automation.md section 13, this step is known to fail on dd-nas1 - it is always run (not gated behind --include-optional) so evidence is collected on every pass, and a failure is RECORDED rather than blocking Part 8 or the run's exit code. See run-tests' followups list. Assertions below encode the CORRECT/expected behaviour per TESTING.md; if the underlying bug is ever fixed this step will show PASS.",
      "actions": [
        { "do": "zfs_destroy", "dataset": "{DST}/ds1", "recursive": 1, "missing_ok": 1 },
        { "do": "zfs_destroy", "dataset": "{DST}/ds2", "recursive": 1, "missing_ok": 1 },
        { "do": "rm_matching", "dir": "{SN}", "pattern": "^sneakernet_target\\.status(\\.[0-9_.-]+)?$" },
        { "do": "empty_dir", "path": "{TRANSPORT}" },
        { "do": "zfs_snapshot", "dataset": "{SRC}/ds1@part8_5" },
        { "do": "run_sneakernet", "servername": "test-source", "verbosity": 1, "expect_exit": 0 },
        { "do": "list_matching", "dir": "{TRANSPORT}/datasets", "pattern": "^ds1\\.xz$",
          "capture": { "name": "DS1_XZ_85", "reduce": "count" } },
        { "do": "zfs_snapshot", "dataset": "{SRC}/ds2@part8_5" },
        { "do": "random_hex", "bytes": 16, "store_as": "IVHEX85" },
        { "do": "pipeline_send_to_file", "snapshot": "{SRC}/ds2@part8_5", "key": "$TESTKEY", "iv": "$IVHEX85",
          "out_path": "{TRANSPORT}/datasets/ds2" },
        { "do": "write_file", "path": "{TRANSPORT}/datasets/ds2.IV", "content": "$IVHEX85" },
        { "do": "run_sneakernet", "servername": "test-target", "verbosity": 3, "log": "part8_5", "expect_exit": 0,
          "capture": { "name": "LOG85" } },
        { "do": "zfs_list_all", "dataset": "{DST}", "capture": { "name": "DST_ALL_85", "reduce": "join" } }
      ],
      "assertions": [
        { "type": "count_equals", "var": "DS1_XZ_85", "n": 1, "message": "ds1 went through sneakernet normally and came out compressed (.xz)" },
        { "type": "regex_present", "var": "DST_ALL_85", "pattern": "dst/ds1\\@part8_5", "message": "ds1 (from the .xz file) received successfully" },
        { "type": "regex_present", "var": "DST_ALL_85", "pattern": "dst/ds2\\@part8_5", "message": "ds2 (from the hand-added plain file) received successfully in the same target run" }
      ]
    },
    { "id": "8.6", "title": "Restore method: 'off' before continuing to Part 9", "depends_on": ["8.5"],
      "config": { "restore": "none", "mutations": [
        { "op": "set", "path": "transport.compression.method", "value": "off", "style": "single" }
      ] }
    }
  ]
}