From 617032bc0fb425ffc323e3a29269bdd85b8a7ed8 Mon Sep 17 00:00:00 2001
From: Samuel Justus Schwertfeger
 <113480134+SamuelSchwertfeger@users.noreply.github.com>
Date: Fri, 9 Oct 2026 18:15:04 -0400
Subject: [PATCH 1/2] Add named train/test splits

time: for each class in each source file, the latest 20% of the flows by
timestamp are the test set. random: 20% of each class, fixed seed.
The release records its timestamp column and format.
---
 README.md                |  21 ++++++-
 src/idsdata/__init__.py  |  18 ++++++
 src/idsdata/model.py     |   3 +
 src/idsdata/registry.py  |   2 +
 src/idsdata/splitting.py |  67 ++++++++++++++++++++
 tests/test_splitting.py  | 129 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 239 insertions(+), 1 deletion(-)
 create mode 100644 src/idsdata/splitting.py
 create mode 100644 tests/test_splitting.py

diff --git a/README.md b/README.md
index 9ffdeec..077fd7d 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,26 @@ attack = flows['is_attack'] | flows['is_attempted']  # or count them as attacks
 
 The first load reads the CSV files and writes a parquet cache next to them. Later loads read the cache.
 
+## Train/test splits
+
+```python
+train, test = idsdata.split('cic-ids2017', 'engelen2021', 'time')
+```
+
+A split has a name, and the name always means the same rows. Both splits put 20% of the flows in the test set and keep every class on both sides.
+
+| split | how the test set is chosen | use it for |
+|---|---|---|
+| `time` (default) | For each class in each source file, the latest 20% of the flows by timestamp | New results. A model is tested on flows that come after the ones it was trained on. |
+| `random` | 20% of each class, drawn with a fixed seed | Comparison with earlier work that shuffled the rows. Near-duplicate flows land on both sides, so scores are optimistic. |
+
+Things to know:
+
+- The cut is made per class, not once per day. Each attack runs in a short window, so a single cut-off time per day would leave most attack classes out of the test set.
+- Timestamps have one-second resolution. Flows that share the second at the cut keep the order they have in the source file.
+- A class with two or more flows always gets at least one test flow; a class with a single flow stays in train. Some classes are very small, so their test sets are tiny.
+- A split is of the whole release. Filter afterwards, for example `train[~train['is_attempted']]`.
+
 ## Datasets
 
 | name | version | what it is | how you get it |
@@ -109,7 +129,6 @@ The linter reports and changes nothing. A warning is a prompt to decide, and wha
 
 ## Not in this version
 
-- Train/test splits. The split design is still open.
 - Checksums for the `original` release.
 - Other datasets.
 
diff --git a/src/idsdata/__init__.py b/src/idsdata/__init__.py
index 77c7cde..a775036 100644
--- a/src/idsdata/__init__.py
+++ b/src/idsdata/__init__.py
@@ -37,6 +37,7 @@
     'download',
     'info',
     'load',
+    'split',
     'verify',
 ]
 
@@ -76,3 +77,20 @@ def load(
     from idsdata.loading import load as _load
 
     return _load(name, version, data_dir, files)
+
+
+def split(
+    name: str,
+    version: str,
+    split: str = 'time',
+    data_dir: str | os.PathLike[str] | None = None,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Load a whole release and return it as ``(train, test)``.
+
+    ``time``: for each class in each source file, the earliest 80% of the flows
+    are train and the latest 20% are test. ``random``: 20% of each class, drawn
+    with a fixed seed, is test. Rows keep the index and order ``load`` gives them.
+    """
+    from idsdata.splitting import split as _split
+
+    return _split(name, version, split, data_dir)
diff --git a/src/idsdata/model.py b/src/idsdata/model.py
index 0b63e58..5c600f0 100644
--- a/src/idsdata/model.py
+++ b/src/idsdata/model.py
@@ -80,3 +80,6 @@ class Release:
     labels: tuple[Label, ...] = ()
     retrieved: str | None = None
     notes: str = ''
+    # Clean name of the column that holds the flow time, and its strptime format.
+    time_column: str | None = None
+    time_format: str | None = None
diff --git a/src/idsdata/registry.py b/src/idsdata/registry.py
index ddb695d..b2d9685 100644
--- a/src/idsdata/registry.py
+++ b/src/idsdata/registry.py
@@ -71,6 +71,8 @@ class UnknownDatasetError(LookupError):
         files=ENGELEN_FILES,
         columns=ENGELEN_COLUMNS,
         labels=ENGELEN_LABELS,
+        time_column='timestamp',
+        time_format='%d/%m/%Y %I:%M:%S %p',
         retrieved='2026-10-09',
         notes=(
             'The authors re-uploaded the files on 2021-10-20, 2021-10-22 and 2021-11-24; '
diff --git a/src/idsdata/splitting.py b/src/idsdata/splitting.py
new file mode 100644
index 0000000..2e0e225
--- /dev/null
+++ b/src/idsdata/splitting.py
@@ -0,0 +1,67 @@
+"""Named train/test splits of a release."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pandas as pd
+
+from idsdata.loading import load
+from idsdata.registry import get as get_release
+
+if TYPE_CHECKING:
+    import os
+
+    from idsdata.model import Release
+
+SPLITS = ('time', 'random')
+
+# Both numbers are part of what a split name means. Changing either makes a different split.
+TEST_PERCENT = 20
+SEED = 0
+
+
+def _latest(order: pd.Series, groups: list[pd.Series]) -> pd.Series:
+    """Mark the last ``TEST_PERCENT`` percent of each group, in the order given, as test rows.
+
+    Rows that tie in ``order`` keep the order they have in the frame. A group of
+    two or more rows always gets at least one test row; a single row stays in train.
+    """
+    grouped = order.groupby(groups, sort=False)
+    rank = grouped.rank(method='first').astype('int64')
+    size = grouped.transform('size').astype('int64')
+    test_rows = (size * TEST_PERCENT // 100).clip(lower=1).where(size > 1, 0)
+    return rank > size - test_rows
+
+
+def _time_mask(flows: pd.DataFrame, release: Release) -> pd.Series:
+    if release.time_column is None or release.time_format is None:
+        raise ValueError(f'no timestamp column is recorded for {release.name} {release.version}; use the random split')
+    times = pd.to_datetime(flows[release.time_column], format=release.time_format)
+    return _latest(times, [flows['source_file'], flows['label']])
+
+
+def _random_mask(flows: pd.DataFrame) -> pd.Series:
+    shuffled = pd.Series(range(len(flows)), index=flows.index).sample(frac=1, random_state=SEED)
+    position = pd.Series(range(len(flows)), index=shuffled.index).reindex(flows.index)
+    return _latest(position, [flows['label']])
+
+
+def split(
+    name: str,
+    version: str,
+    split: str = 'time',
+    data_dir: str | os.PathLike[str] | None = None,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Load a whole release and return it as ``(train, test)``.
+
+    ``time``: for each class in each source file, the earliest 80% of the flows
+    are train and the latest 20% are test. ``random``: 20% of each class, drawn
+    with a fixed seed, is test. Rows keep the index and order ``load`` gives them.
+    """
+    release = get_release(name, version)
+    if split not in SPLITS:
+        raise ValueError(f'unknown split {split!r}; known splits: {", ".join(SPLITS)}')
+    flows = load(name, version, data_dir)
+    test = _time_mask(flows, release) if split == 'time' else _random_mask(flows)
+    return flows.loc[~test, :], flows.loc[test, :]
diff --git a/tests/test_splitting.py b/tests/test_splitting.py
new file mode 100644
index 0000000..ec17407
--- /dev/null
+++ b/tests/test_splitting.py
@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import dataclasses
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+import idsdata
+
+from idsdata.model import Column, Label
+from idsdata.splitting import SPLITS
+from tests.conftest import make_release, register
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+# Everything below is made up for the tests. The row number is the Count column.
+ONE = b"""Timestamp,Count,Label
+03/07/2017 01:00:09 PM,0,BENIGN
+03/07/2017 01:00:01 PM,1,BENIGN
+03/07/2017 12:59:59 PM,2,BENIGN
+03/07/2017 01:00:05 PM,3,Probe
+03/07/2017 01:00:03 PM,4,BENIGN
+03/07/2017 01:00:09 PM,5,BENIGN
+03/07/2017 01:00:02 PM,6,Probe
+03/07/2017 11:00:00 AM,7,BENIGN
+03/07/2017 01:00:04 PM,8,BENIGN
+03/07/2017 01:00:06 PM,9,BENIGN
+03/07/2017 01:00:07 PM,10,BENIGN
+03/07/2017 01:00:09 PM,11,BENIGN
+03/07/2017 01:00:08 PM,12,Probe
+03/07/2017 01:00:01 PM,13,Probe
+03/07/2017 01:00:04 PM,14,Probe
+03/07/2017 01:00:00 PM,15,Rare
+"""
+TWO = b"""Timestamp,Count,Label
+04/07/2017 09:00:00 AM,16,BENIGN
+04/07/2017 09:00:02 AM,17,BENIGN
+04/07/2017 09:00:01 AM,18,BENIGN
+"""
+FILES = {'one.csv': ONE, 'two.csv': TWO}
+COLUMNS = (
+    Column('Timestamp', 'timestamp', 'string'),
+    Column('Count', 'count', 'int64'),
+    Column('Label', 'label_raw', 'string'),
+)
+PINNED = [0, 11, 12]
+LABELS = (
+    Label('BENIGN', 'benign', is_attack=False),
+    Label('Probe', 'probe', is_attack=True),
+    Label('Rare', 'rare', is_attack=True),
+)
+
+
+@pytest.fixture
+def timed(data_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    """A release with a timestamp column, placed in the data directory."""
+    release = dataclasses.replace(
+        make_release('v1', FILES),
+        columns=COLUMNS,
+        labels=LABELS,
+        time_column='timestamp',
+        time_format='%d/%m/%Y %I:%M:%S %p',
+    )
+    register(monkeypatch, release, dataclasses.replace(release, version='untimed', time_column=None, time_format=None))
+    for version in ('v1', 'untimed'):
+        directory = data_dir / 'synthetic' / version
+        directory.mkdir(parents=True)
+        for name, content in FILES.items():
+            _ = (directory / name).write_bytes(content)
+    return data_dir
+
+
+def test_time_split_takes_the_latest_flows_of_each_class_in_each_file(timed: Path) -> None:
+    train, test = idsdata.split('synthetic', 'v1', 'time', timed)
+    # benign in one.csv: rows 0, 5 and 11 share the latest second and the first of them stays in train.
+    # probe in one.csv: row 12 is the latest. benign in two.csv: three rows still give one test row.
+    assert test['count'].tolist() == [5, 11, 12, 17]
+    assert train['count'].tolist() == [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 13, 14, 15, 16, 18]
+
+
+def test_time_is_the_default_split(timed: Path) -> None:
+    assert idsdata.split('synthetic', 'v1', data_dir=timed)[1]['count'].tolist() == [5, 11, 12, 17]
+
+
+def test_a_class_with_one_row_stays_in_train(timed: Path) -> None:
+    for name in SPLITS:
+        train, test = idsdata.split('synthetic', 'v1', name, timed)
+        assert 'rare' in set(train['label'])
+        assert 'rare' not in set(test['label'])
+
+
+def test_random_split_is_stratified_and_repeatable(timed: Path) -> None:
+    train, test = idsdata.split('synthetic', 'v1', 'random', timed)
+    assert test['label'].value_counts().to_dict() == {'benign': 2, 'probe': 1}
+    assert sorted([*train['count'], *test['count']]) == list(range(19))
+    again = idsdata.split('synthetic', 'v1', 'random', timed)[1]
+    assert again['count'].tolist() == test['count'].tolist()
+
+
+def test_random_split_does_not_drift(timed: Path) -> None:
+    # Pinned so that a change in pandas or in the code that moves rows between the sides is noticed.
+    assert idsdata.split('synthetic', 'v1', 'random', timed)[1]['count'].tolist() == PINNED
+
+
+def test_splits_keep_the_index_and_columns_of_load(timed: Path) -> None:
+    flows = idsdata.load('synthetic', 'v1', timed)
+    train, test = idsdata.split('synthetic', 'v1', 'time', timed)
+    assert list(train.columns) == list(flows.columns)
+    assert train.index.tolist() == train['count'].tolist()
+    assert test.index.tolist() == test['count'].tolist()
+
+
+def test_time_split_needs_a_recorded_timestamp_column(timed: Path) -> None:
+    with pytest.raises(ValueError, match='no timestamp column is recorded for synthetic untimed'):
+        _ = idsdata.split('synthetic', 'untimed', 'time', timed)
+    assert len(idsdata.split('synthetic', 'untimed', 'random', timed)[1]) == 3
+
+
+def test_unknown_split_names_are_rejected(timed: Path) -> None:
+    with pytest.raises(ValueError, match='known splits: time, random'):
+        _ = idsdata.split('synthetic', 'v1', 'chronological', timed)
+
+
+def test_the_real_release_records_its_timestamp_column() -> None:
+    release = idsdata.info('cic-ids2017', 'engelen2021')
+    assert release.time_column in {column.name for column in release.columns}
+    assert release.time_format

From 15698f7907897174605a75ff351e2d3cf593980d Mon Sep 17 00:00:00 2001
From: Samuel Justus Schwertfeger
 <113480134+SamuelSchwertfeger@users.noreply.github.com>
Date: Fri, 9 Oct 2026 18:35:26 -0400
Subject: [PATCH 2/2] Record the original CIC-IDS2017 release as original-flows
 and original-ml

Both versions are recorded from the files downloaded through the
publisher's form. The loader learns archive member paths, a per-release
text encoding, repeated column names and recorded blank lines, and checks
the row count of every file. ids-lint reads a file in the encoding of the
release and no longer fails on a console that cannot show a label.
The README says how the package was tested.
---
 README.md                            |  38 ++-
 src/idsdata/_cic_ids2017_original.py | 360 +++++++++++++++++++++++++++
 src/idsdata/lint.py                  |  23 +-
 src/idsdata/loading.py               |  41 ++-
 src/idsdata/model.py                 |  11 +-
 src/idsdata/registry.py              |  55 +++-
 tests/test_cli.py                    |  10 +-
 tests/test_downloading.py            |   5 +-
 tests/test_lint.py                   |   7 +
 tests/test_loading.py                |  11 +-
 tests/test_original.py               | 135 ++++++++++
 tests/test_registry.py               |  13 +-
 12 files changed, 671 insertions(+), 38 deletions(-)
 create mode 100644 src/idsdata/_cic_ids2017_original.py
 create mode 100644 tests/test_original.py

diff --git a/README.md b/README.md
index 077fd7d..9e737bb 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@ print(flows['label'].value_counts())
 print(idsdata.cite('cic-ids2017', 'engelen2021'))
 ```
 
-`load` keeps every row and every value as it is in the source files. It gives the columns clean names (`Flow Bytes/s` becomes `flow_bytes_s`), keeps the source label as `label_raw`, and adds four columns:
+`load` keeps every flow and every value as it is in the source files. It gives the columns clean names (`Flow Bytes/s` becomes `flow_bytes_s`), keeps the source label as `label_raw`, and adds four columns:
 
 | column | meaning |
 |---|---|
@@ -75,9 +75,17 @@ Things to know:
 | name | version | what it is | how you get it |
 |---|---|---|---|
 | `cic-ids2017` | `engelen2021` | Corrected flows and labels by Engelen, Rimmer and Joosen | `idsdata download`, or by hand from the authors' page |
-| `cic-ids2017` | `original` | The original release from the Canadian Institute for Cybersecurity | By hand, through the request form on the publisher's page |
+| `cic-ids2017` | `original-flows` | The original release from the Canadian Institute for Cybersecurity, with flow ids, addresses and timestamps (`GeneratedLabelledFlows.zip`) | By hand, through the request form on the publisher's page |
+| `cic-ids2017` | `original-ml` | The same flows without ids, addresses and timestamps (`MachineLearningCSV.zip`) | By hand, through the same form |
 
-No file hashes are recorded for `original` yet, so it cannot be verified or loaded. `idsdata info cic-ids2017 original` prints where to get it and how to cite it.
+For the two original versions, fill in the form, take the zip from the `CSVs` folder and put it in the directory `idsdata where cic-ids2017 original-flows` (or `original-ml`) prints. `verify` checks the zip and `load` unpacks it.
+
+Both original versions hold the same 2,830,743 flows in 15 classes. They are kept as published, and four things about them are worth knowing:
+
+- The column ` Fwd Header Length` appears twice in every file. Both are kept; the second is named `fwd_header_length_1`. In these files the two hold the same values.
+- One file of `original-flows` holds 288,602 lines without any value. `load` leaves them out. This is the only case where lines of a source file are not returned, and the number is recorded and checked.
+- The dash in the three web attack labels is not plain text. `original-flows` is read as Windows-1252, which makes it an en dash; in `original-ml` it is the Unicode replacement character. `label_raw` shows what the file holds, and `label` is the same in both (`web_attack_xss`).
+- The timestamps of `original-flows` come in more than one format and without AM or PM, so only the `random` split is offered for the original versions.
 
 ## Data policy
 
@@ -110,7 +118,7 @@ ids-lint --release cic-ids2017 engelen2021 *.csv
 ids-lint --rules
 ```
 
-The exit status is 1 when a rule of severity `error` fires. With `--strict`, warnings count as well. `--release` also checks that every label string in the file belongs to that release.
+The exit status is 1 when a rule of severity `error` fires. With `--strict`, warnings count as well. `--release` also checks that every label string in the file belongs to that release, and reads the file in the text encoding recorded for it.
 
 <!-- rules:start -->
 | id | rule | severity | what it flags | source |
@@ -127,9 +135,29 @@ The exit status is 1 when a rule of severity `error` fires. With `--strict`, war
 
 The linter reports and changes nothing. A warning is a prompt to decide, and what to do with the flagged rows or columns is up to you.
 
+## How it was tested
+
+- **Unit tests.** The test suite runs on every pull request, on Python 3.10 and 3.12 (GitHub Actions, Linux), together with `ruff` and `basedpyright`. They use small made-up files only, never real data: checksums, the download rules, loading, the cache, label mapping, both splits and every lint rule. The `random` split is pinned to exact rows, so a change in pandas that moves rows between train and test is noticed. One test checks that the rule table above matches the code.
+- **Real data, by hand.** The numbers in the registry (hashes, sizes, row counts, column names, label strings) were read from real downloads made on 2026-10-09, not typed in. The MD5 files the publisher ships with the original release match the two zips. These checks were then run on Windows 11 with Python 3.12, and are not part of CI because the data cannot be redistributed:
+
+| check | `engelen2021` | `original-flows` | `original-ml` |
+|---|---|---|---|
+| `verify` passes on the downloaded archive | yes | yes | yes |
+| `load` straight from the archive | 2,100,814 rows, 88 columns, 25 classes | 2,830,743 rows, 89 columns, 15 classes | 2,830,743 rows, 83 columns, 15 classes |
+| rows per file equal the recorded counts | yes | yes | yes |
+| `split(..., 'time')` train / test | 1,680,663 / 420,151 | not offered | not offered |
+| `split(..., 'random')` train / test | 1,680,661 / 420,153 | 2,264,600 / 566,143 | 2,264,600 / 566,143 |
+| every class on both sides of each split | yes | yes | yes |
+| `ids-lint --release` on every file: unknown label strings | none | none | none |
+| `ids-lint` rules that fire | IDS002, IDS004, IDS005, IDS006 | all six | all but IDS003 |
+
+The class counts of the two original versions are equal label by label.
+
+What was not tested: other operating systems and Python versions with real data, and `idsdata download` beyond the one release that has a direct link.
+
 ## Not in this version
 
-- Checksums for the `original` release.
+- A time split for the original versions.
 - Other datasets.
 
 ## Acknowledgements
diff --git a/src/idsdata/_cic_ids2017_original.py b/src/idsdata/_cic_ids2017_original.py
new file mode 100644
index 0000000..53639fb
--- /dev/null
+++ b/src/idsdata/_cic_ids2017_original.py
@@ -0,0 +1,360 @@
+"""The original CIC-IDS2017 release, in the two forms the publisher offers.
+
+Every value below was read from the files downloaded through the
+publisher's request form on 2026-10-09.
+"""
+
+from idsdata.model import Column, FileEntry, Label
+
+FLOWS_FILES = (
+    FileEntry(
+        name='GeneratedLabelledFlows.zip',
+        sha256='7bdbef286f8893f31c6db12105fa097fa5c2dcc6733179037a08129d150ea27a',
+        size=283876488,
+        kind='archive',
+    ),
+    FileEntry(
+        name='Monday-WorkingHours.pcap_ISCX.csv',
+        sha256='96f26aea87d513073769b48ce204c2921791149e1c6225f7db65a3fed7973820',
+        size=268649908,
+        archive='GeneratedLabelledFlows.zip',
+        member='TrafficLabelling /Monday-WorkingHours.pcap_ISCX.csv',
+        rows=529918,
+    ),
+    FileEntry(
+        name='Tuesday-WorkingHours.pcap_ISCX.csv',
+        sha256='ae9c88e10c41a8eb1ff454ae98bc513454925097d0b0b57180f94e79de445815',
+        size=174696560,
+        archive='GeneratedLabelledFlows.zip',
+        member='TrafficLabelling /Tuesday-WorkingHours.pcap_ISCX.csv',
+        rows=445909,
+    ),
+    FileEntry(
+        name='Wednesday-workingHours.pcap_ISCX.csv',
+        sha256='ed538e85b84181e8897dedb3d37d365982f44b27eccd67c477581a8b65f3d170',
+        size=285642925,
+        archive='GeneratedLabelledFlows.zip',
+        member='TrafficLabelling /Wednesday-workingHours.pcap_ISCX.csv',
+        rows=692703,
+    ),
+    FileEntry(
+        name='Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv',
+        sha256='e3deaff483d18b53b100a441dff9b8919416df8876b4edcc0f0d1e4bf48397d0',
+        size=92030223,
+        archive='GeneratedLabelledFlows.zip',
+        member='TrafficLabelling /Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv',
+        rows=170366,
+        blank_rows=288602,
+    ),
+    FileEntry(
+        name='Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv',
+        sha256='d74238e054023c8bd4d8056650463cadb25c100fb0e26ebd4272c07b5c022c4b',
+        size=108723183,
+        archive='GeneratedLabelledFlows.zip',
+        member='TrafficLabelling /Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv',
+        rows=288602,
+    ),
+    FileEntry(
+        name='Friday-WorkingHours-Morning.pcap_ISCX.csv',
+        sha256='c061cd98f39d8054aeaed6244a5129a20b3985983d0d022b860ae8aa07fe8d21',
+        size=75386737,
+        archive='GeneratedLabelledFlows.zip',
+        member='TrafficLabelling /Friday-WorkingHours-Morning.pcap_ISCX.csv',
+        rows=191033,
+    ),
+    FileEntry(
+        name='Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv',
+        sha256='7e2ddaa80a5849ba629463296b6128436c161b4a32e8034bb5beae06b0c45e08',
+        size=101874777,
+        archive='GeneratedLabelledFlows.zip',
+        member='TrafficLabelling /Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv',
+        rows=286467,
+    ),
+    FileEntry(
+        name='Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv',
+        sha256='1f779b4f0d78f9225554c4de53b5a2c07912b60dcd136ee4c5c1d0d2496b7cc4',
+        size=96101069,
+        archive='GeneratedLabelledFlows.zip',
+        member='TrafficLabelling /Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv',
+        rows=225745,
+    ),
+)
+
+FLOWS_COLUMNS = (
+    Column('Flow ID', 'flow_id', 'string'),
+    Column(' Source IP', 'source_ip', 'string'),
+    Column(' Source Port', 'source_port', 'int64'),
+    Column(' Destination IP', 'destination_ip', 'string'),
+    Column(' Destination Port', 'destination_port', 'int64'),
+    Column(' Protocol', 'protocol', 'int64'),
+    Column(' Timestamp', 'timestamp', 'string'),
+    Column(' Flow Duration', 'flow_duration', 'int64'),
+    Column(' Total Fwd Packets', 'total_fwd_packets', 'int64'),
+    Column(' Total Backward Packets', 'total_backward_packets', 'int64'),
+    Column('Total Length of Fwd Packets', 'total_length_of_fwd_packets', 'double'),
+    Column(' Total Length of Bwd Packets', 'total_length_of_bwd_packets', 'double'),
+    Column(' Fwd Packet Length Max', 'fwd_packet_length_max', 'double'),
+    Column(' Fwd Packet Length Min', 'fwd_packet_length_min', 'double'),
+    Column(' Fwd Packet Length Mean', 'fwd_packet_length_mean', 'double'),
+    Column(' Fwd Packet Length Std', 'fwd_packet_length_std', 'double'),
+    Column('Bwd Packet Length Max', 'bwd_packet_length_max', 'double'),
+    Column(' Bwd Packet Length Min', 'bwd_packet_length_min', 'double'),
+    Column(' Bwd Packet Length Mean', 'bwd_packet_length_mean', 'double'),
+    Column(' Bwd Packet Length Std', 'bwd_packet_length_std', 'double'),
+    Column('Flow Bytes/s', 'flow_bytes_s', 'double'),
+    Column(' Flow Packets/s', 'flow_packets_s', 'double'),
+    Column(' Flow IAT Mean', 'flow_iat_mean', 'double'),
+    Column(' Flow IAT Std', 'flow_iat_std', 'double'),
+    Column(' Flow IAT Max', 'flow_iat_max', 'double'),
+    Column(' Flow IAT Min', 'flow_iat_min', 'double'),
+    Column('Fwd IAT Total', 'fwd_iat_total', 'double'),
+    Column(' Fwd IAT Mean', 'fwd_iat_mean', 'double'),
+    Column(' Fwd IAT Std', 'fwd_iat_std', 'double'),
+    Column(' Fwd IAT Max', 'fwd_iat_max', 'double'),
+    Column(' Fwd IAT Min', 'fwd_iat_min', 'double'),
+    Column('Bwd IAT Total', 'bwd_iat_total', 'double'),
+    Column(' Bwd IAT Mean', 'bwd_iat_mean', 'double'),
+    Column(' Bwd IAT Std', 'bwd_iat_std', 'double'),
+    Column(' Bwd IAT Max', 'bwd_iat_max', 'double'),
+    Column(' Bwd IAT Min', 'bwd_iat_min', 'double'),
+    Column('Fwd PSH Flags', 'fwd_psh_flags', 'int64'),
+    Column(' Bwd PSH Flags', 'bwd_psh_flags', 'int64'),
+    Column(' Fwd URG Flags', 'fwd_urg_flags', 'int64'),
+    Column(' Bwd URG Flags', 'bwd_urg_flags', 'int64'),
+    Column(' Fwd Header Length', 'fwd_header_length', 'int64'),
+    Column(' Bwd Header Length', 'bwd_header_length', 'int64'),
+    Column('Fwd Packets/s', 'fwd_packets_s', 'double'),
+    Column(' Bwd Packets/s', 'bwd_packets_s', 'double'),
+    Column(' Min Packet Length', 'min_packet_length', 'double'),
+    Column(' Max Packet Length', 'max_packet_length', 'double'),
+    Column(' Packet Length Mean', 'packet_length_mean', 'double'),
+    Column(' Packet Length Std', 'packet_length_std', 'double'),
+    Column(' Packet Length Variance', 'packet_length_variance', 'double'),
+    Column('FIN Flag Count', 'fin_flag_count', 'int64'),
+    Column(' SYN Flag Count', 'syn_flag_count', 'int64'),
+    Column(' RST Flag Count', 'rst_flag_count', 'int64'),
+    Column(' PSH Flag Count', 'psh_flag_count', 'int64'),
+    Column(' ACK Flag Count', 'ack_flag_count', 'int64'),
+    Column(' URG Flag Count', 'urg_flag_count', 'int64'),
+    Column(' CWE Flag Count', 'cwe_flag_count', 'int64'),
+    Column(' ECE Flag Count', 'ece_flag_count', 'int64'),
+    Column(' Down/Up Ratio', 'down_up_ratio', 'double'),
+    Column(' Average Packet Size', 'average_packet_size', 'double'),
+    Column(' Avg Fwd Segment Size', 'avg_fwd_segment_size', 'double'),
+    Column(' Avg Bwd Segment Size', 'avg_bwd_segment_size', 'double'),
+    Column(' Fwd Header Length', 'fwd_header_length_1', 'int64'),
+    Column('Fwd Avg Bytes/Bulk', 'fwd_avg_bytes_bulk', 'int64'),
+    Column(' Fwd Avg Packets/Bulk', 'fwd_avg_packets_bulk', 'int64'),
+    Column(' Fwd Avg Bulk Rate', 'fwd_avg_bulk_rate', 'int64'),
+    Column(' Bwd Avg Bytes/Bulk', 'bwd_avg_bytes_bulk', 'int64'),
+    Column(' Bwd Avg Packets/Bulk', 'bwd_avg_packets_bulk', 'int64'),
+    Column('Bwd Avg Bulk Rate', 'bwd_avg_bulk_rate', 'int64'),
+    Column('Subflow Fwd Packets', 'subflow_fwd_packets', 'int64'),
+    Column(' Subflow Fwd Bytes', 'subflow_fwd_bytes', 'int64'),
+    Column(' Subflow Bwd Packets', 'subflow_bwd_packets', 'int64'),
+    Column(' Subflow Bwd Bytes', 'subflow_bwd_bytes', 'int64'),
+    Column('Init_Win_bytes_forward', 'init_win_bytes_forward', 'int64'),
+    Column(' Init_Win_bytes_backward', 'init_win_bytes_backward', 'int64'),
+    Column(' act_data_pkt_fwd', 'act_data_pkt_fwd', 'int64'),
+    Column(' min_seg_size_forward', 'min_seg_size_forward', 'int64'),
+    Column('Active Mean', 'active_mean', 'double'),
+    Column(' Active Std', 'active_std', 'double'),
+    Column(' Active Max', 'active_max', 'double'),
+    Column(' Active Min', 'active_min', 'double'),
+    Column('Idle Mean', 'idle_mean', 'double'),
+    Column(' Idle Std', 'idle_std', 'double'),
+    Column(' Idle Max', 'idle_max', 'double'),
+    Column(' Idle Min', 'idle_min', 'double'),
+    Column(' Label', 'label_raw', 'string'),
+)
+
+FLOWS_LABELS = (
+    Label('BENIGN', 'benign', is_attack=False),
+    Label('Bot', 'bot', is_attack=True),
+    Label('DDoS', 'ddos', is_attack=True),
+    Label('DoS GoldenEye', 'dos_goldeneye', is_attack=True),
+    Label('DoS Hulk', 'dos_hulk', is_attack=True),
+    Label('DoS Slowhttptest', 'dos_slowhttptest', is_attack=True),
+    Label('DoS slowloris', 'dos_slowloris', is_attack=True),
+    Label('FTP-Patator', 'ftp_patator', is_attack=True),
+    Label('Heartbleed', 'heartbleed', is_attack=True),
+    Label('Infiltration', 'infiltration', is_attack=True),
+    Label('PortScan', 'portscan', is_attack=True),
+    Label('SSH-Patator', 'ssh_patator', is_attack=True),
+    Label('Web Attack \u2013 Brute Force', 'web_attack_brute_force', is_attack=True),
+    Label('Web Attack \u2013 Sql Injection', 'web_attack_sql_injection', is_attack=True),
+    Label('Web Attack \u2013 XSS', 'web_attack_xss', is_attack=True),
+)
+
+ML_FILES = (
+    FileEntry(
+        name='MachineLearningCSV.zip',
+        sha256='c3f26274b36c837ccf28ffd2dbf4582941c30b3ee70a635c6e5b2f87c4727928',
+        size=235102953,
+        kind='archive',
+    ),
+    FileEntry(
+        name='Monday-WorkingHours.pcap_ISCX.csv',
+        sha256='852c4beb34eda186f32561fa79df7a0747e92e1a6535b01270820dd9ffe17f34',
+        size=176927918,
+        archive='MachineLearningCSV.zip',
+        member='MachineLearningCVE/Monday-WorkingHours.pcap_ISCX.csv',
+        rows=529918,
+    ),
+    FileEntry(
+        name='Tuesday-WorkingHours.pcap_ISCX.csv',
+        sha256='52b8692ae8c7d2ed04671fe2b98335693c0a92c7ab157d8c8b534d6523080851',
+        size=135078995,
+        archive='MachineLearningCSV.zip',
+        member='MachineLearningCVE/Tuesday-WorkingHours.pcap_ISCX.csv',
+        rows=445909,
+    ),
+    FileEntry(
+        name='Wednesday-workingHours.pcap_ISCX.csv',
+        sha256='893c27dc968bf7a8adef1689f90be55ca4a4dc3088fb63d6ff247ac56856df2a',
+        size=225166395,
+        archive='MachineLearningCSV.zip',
+        member='MachineLearningCVE/Wednesday-workingHours.pcap_ISCX.csv',
+        rows=692703,
+    ),
+    FileEntry(
+        name='Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv',
+        sha256='d67066211fb1689c78406f1506f4c44704ecb92088353d5c96d96d6474eb819d',
+        size=52023263,
+        archive='MachineLearningCSV.zip',
+        member='MachineLearningCVE/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv',
+        rows=170366,
+    ),
+    FileEntry(
+        name='Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv',
+        sha256='6bcda3857c2504676034e3ea57762d38393cc734cb377a726bd5cb153961b1b5',
+        size=83102436,
+        archive='MachineLearningCSV.zip',
+        member='MachineLearningCVE/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv',
+        rows=288602,
+    ),
+    FileEntry(
+        name='Friday-WorkingHours-Morning.pcap_ISCX.csv',
+        sha256='53a41c24d570ea83b7ac55b2e94df94e7a8216aeb80a2af0246b6bc8bb543000',
+        size=58316725,
+        archive='MachineLearningCSV.zip',
+        member='MachineLearningCVE/Friday-WorkingHours-Morning.pcap_ISCX.csv',
+        rows=191033,
+    ),
+    FileEntry(
+        name='Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv',
+        sha256='ca1824c51bfbb7b3c72290a11be04366ba8815878c6a1cc5c44cb1cee269e99b',
+        size=76906168,
+        archive='MachineLearningCSV.zip',
+        member='MachineLearningCVE/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv',
+        rows=286467,
+    ),
+    FileEntry(
+        name='Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv',
+        sha256='6ff1580f5f81c0ae28a26f7631721018577f5f7c5e0feac28b795fcfe7b411ee',
+        size=77123859,
+        archive='MachineLearningCSV.zip',
+        member='MachineLearningCVE/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv',
+        rows=225745,
+    ),
+)
+
+ML_COLUMNS = (
+    Column(' Destination Port', 'destination_port', 'int64'),
+    Column(' Flow Duration', 'flow_duration', 'int64'),
+    Column(' Total Fwd Packets', 'total_fwd_packets', 'int64'),
+    Column(' Total Backward Packets', 'total_backward_packets', 'int64'),
+    Column('Total Length of Fwd Packets', 'total_length_of_fwd_packets', 'int64'),
+    Column(' Total Length of Bwd Packets', 'total_length_of_bwd_packets', 'int64'),
+    Column(' Fwd Packet Length Max', 'fwd_packet_length_max', 'int64'),
+    Column(' Fwd Packet Length Min', 'fwd_packet_length_min', 'int64'),
+    Column(' Fwd Packet Length Mean', 'fwd_packet_length_mean', 'double'),
+    Column(' Fwd Packet Length Std', 'fwd_packet_length_std', 'double'),
+    Column('Bwd Packet Length Max', 'bwd_packet_length_max', 'int64'),
+    Column(' Bwd Packet Length Min', 'bwd_packet_length_min', 'int64'),
+    Column(' Bwd Packet Length Mean', 'bwd_packet_length_mean', 'double'),
+    Column(' Bwd Packet Length Std', 'bwd_packet_length_std', 'double'),
+    Column('Flow Bytes/s', 'flow_bytes_s', 'double'),
+    Column(' Flow Packets/s', 'flow_packets_s', 'double'),
+    Column(' Flow IAT Mean', 'flow_iat_mean', 'double'),
+    Column(' Flow IAT Std', 'flow_iat_std', 'double'),
+    Column(' Flow IAT Max', 'flow_iat_max', 'int64'),
+    Column(' Flow IAT Min', 'flow_iat_min', 'int64'),
+    Column('Fwd IAT Total', 'fwd_iat_total', 'int64'),
+    Column(' Fwd IAT Mean', 'fwd_iat_mean', 'double'),
+    Column(' Fwd IAT Std', 'fwd_iat_std', 'double'),
+    Column(' Fwd IAT Max', 'fwd_iat_max', 'int64'),
+    Column(' Fwd IAT Min', 'fwd_iat_min', 'int64'),
+    Column('Bwd IAT Total', 'bwd_iat_total', 'int64'),
+    Column(' Bwd IAT Mean', 'bwd_iat_mean', 'double'),
+    Column(' Bwd IAT Std', 'bwd_iat_std', 'double'),
+    Column(' Bwd IAT Max', 'bwd_iat_max', 'int64'),
+    Column(' Bwd IAT Min', 'bwd_iat_min', 'int64'),
+    Column('Fwd PSH Flags', 'fwd_psh_flags', 'int64'),
+    Column(' Bwd PSH Flags', 'bwd_psh_flags', 'int64'),
+    Column(' Fwd URG Flags', 'fwd_urg_flags', 'int64'),
+    Column(' Bwd URG Flags', 'bwd_urg_flags', 'int64'),
+    Column(' Fwd Header Length', 'fwd_header_length', 'int64'),
+    Column(' Bwd Header Length', 'bwd_header_length', 'int64'),
+    Column('Fwd Packets/s', 'fwd_packets_s', 'double'),
+    Column(' Bwd Packets/s', 'bwd_packets_s', 'double'),
+    Column(' Min Packet Length', 'min_packet_length', 'int64'),
+    Column(' Max Packet Length', 'max_packet_length', 'int64'),
+    Column(' Packet Length Mean', 'packet_length_mean', 'double'),
+    Column(' Packet Length Std', 'packet_length_std', 'double'),
+    Column(' Packet Length Variance', 'packet_length_variance', 'double'),
+    Column('FIN Flag Count', 'fin_flag_count', 'int64'),
+    Column(' SYN Flag Count', 'syn_flag_count', 'int64'),
+    Column(' RST Flag Count', 'rst_flag_count', 'int64'),
+    Column(' PSH Flag Count', 'psh_flag_count', 'int64'),
+    Column(' ACK Flag Count', 'ack_flag_count', 'int64'),
+    Column(' URG Flag Count', 'urg_flag_count', 'int64'),
+    Column(' CWE Flag Count', 'cwe_flag_count', 'int64'),
+    Column(' ECE Flag Count', 'ece_flag_count', 'int64'),
+    Column(' Down/Up Ratio', 'down_up_ratio', 'int64'),
+    Column(' Average Packet Size', 'average_packet_size', 'double'),
+    Column(' Avg Fwd Segment Size', 'avg_fwd_segment_size', 'double'),
+    Column(' Avg Bwd Segment Size', 'avg_bwd_segment_size', 'double'),
+    Column(' Fwd Header Length', 'fwd_header_length_1', 'int64'),
+    Column('Fwd Avg Bytes/Bulk', 'fwd_avg_bytes_bulk', 'int64'),
+    Column(' Fwd Avg Packets/Bulk', 'fwd_avg_packets_bulk', 'int64'),
+    Column(' Fwd Avg Bulk Rate', 'fwd_avg_bulk_rate', 'int64'),
+    Column(' Bwd Avg Bytes/Bulk', 'bwd_avg_bytes_bulk', 'int64'),
+    Column(' Bwd Avg Packets/Bulk', 'bwd_avg_packets_bulk', 'int64'),
+    Column('Bwd Avg Bulk Rate', 'bwd_avg_bulk_rate', 'int64'),
+    Column('Subflow Fwd Packets', 'subflow_fwd_packets', 'int64'),
+    Column(' Subflow Fwd Bytes', 'subflow_fwd_bytes', 'int64'),
+    Column(' Subflow Bwd Packets', 'subflow_bwd_packets', 'int64'),
+    Column(' Subflow Bwd Bytes', 'subflow_bwd_bytes', 'int64'),
+    Column('Init_Win_bytes_forward', 'init_win_bytes_forward', 'int64'),
+    Column(' Init_Win_bytes_backward', 'init_win_bytes_backward', 'int64'),
+    Column(' act_data_pkt_fwd', 'act_data_pkt_fwd', 'int64'),
+    Column(' min_seg_size_forward', 'min_seg_size_forward', 'int64'),
+    Column('Active Mean', 'active_mean', 'double'),
+    Column(' Active Std', 'active_std', 'double'),
+    Column(' Active Max', 'active_max', 'int64'),
+    Column(' Active Min', 'active_min', 'int64'),
+    Column('Idle Mean', 'idle_mean', 'double'),
+    Column(' Idle Std', 'idle_std', 'double'),
+    Column(' Idle Max', 'idle_max', 'int64'),
+    Column(' Idle Min', 'idle_min', 'int64'),
+    Column(' Label', 'label_raw', 'string'),
+)
+
+ML_LABELS = (
+    Label('BENIGN', 'benign', is_attack=False),
+    Label('Bot', 'bot', is_attack=True),
+    Label('DDoS', 'ddos', is_attack=True),
+    Label('DoS GoldenEye', 'dos_goldeneye', is_attack=True),
+    Label('DoS Hulk', 'dos_hulk', is_attack=True),
+    Label('DoS Slowhttptest', 'dos_slowhttptest', is_attack=True),
+    Label('DoS slowloris', 'dos_slowloris', is_attack=True),
+    Label('FTP-Patator', 'ftp_patator', is_attack=True),
+    Label('Heartbleed', 'heartbleed', is_attack=True),
+    Label('Infiltration', 'infiltration', is_attack=True),
+    Label('PortScan', 'portscan', is_attack=True),
+    Label('SSH-Patator', 'ssh_patator', is_attack=True),
+    Label('Web Attack \ufffd Brute Force', 'web_attack_brute_force', is_attack=True),
+    Label('Web Attack \ufffd Sql Injection', 'web_attack_sql_injection', is_attack=True),
+    Label('Web Attack \ufffd XSS', 'web_attack_xss', is_attack=True),
+)
diff --git a/src/idsdata/lint.py b/src/idsdata/lint.py
index bdefdd2..d1ea425 100644
--- a/src/idsdata/lint.py
+++ b/src/idsdata/lint.py
@@ -155,8 +155,8 @@ def _listed(names: Sequence[str]) -> str:
     return shown if len(names) <= _SHOWN else f'{shown} and {len(names) - _SHOWN} more'
 
 
-def _read_header(path: Path) -> list[str]:
-    with path.open(encoding='utf-8-sig', errors='replace', newline='') as handle:
+def _read_header(path: Path, encoding: str) -> list[str]:
+    with path.open(encoding=encoding, errors='replace', newline='') as handle:
         return next(csv.reader(handle), [])
 
 
@@ -196,15 +196,18 @@ def _labels(values: pd.Series, known: frozenset[str] | None) -> list[str]:
     return details
 
 
-def lint(path: str | os.PathLike[str], labels: Sequence[str] | None = None) -> list[Finding]:
+def lint(
+    path: str | os.PathLike[str], labels: Sequence[str] | None = None, encoding: str = 'utf-8-sig'
+) -> list[Finding]:
     """Check one CSV file and return what was found, in rule order.
 
     ``labels`` is the set of label strings the file is allowed to contain. When
-    it is left out, only missing labels are reported.
+    it is left out, only missing labels are reported. Bytes that ``encoding``
+    cannot decode are read as the replacement character.
     """
     source = Path(path)
-    header = _read_header(source)
-    frame = pd.read_csv(source, encoding='utf-8', encoding_errors='replace', low_memory=False)
+    header = _read_header(source, encoding)
+    frame = pd.read_csv(source, encoding=encoding, encoding_errors='replace', low_memory=False)
     label_positions = [position for position, name in enumerate(header) if clean_name(name) == 'label']
     label_position = label_positions[-1] if label_positions else None
 
@@ -287,6 +290,7 @@ def main(argv: Sequence[str] | None = None) -> int:
     if not args.files:
         parser.error('give at least one CSV file')
     labels: list[str] | None = None
+    encoding = 'utf-8-sig'
     if args.release is not None:
         try:
             release = get_release(args.release[0], args.release[1])
@@ -297,14 +301,17 @@ def main(argv: Sequence[str] | None = None) -> int:
             print(f'ids-lint: no label strings are recorded for {release.name} {release.version} yet', file=sys.stderr)
             return 2
         labels = [label.raw for label in release.labels]
+        encoding = 'utf-8-sig' if release.encoding == 'utf8' else release.encoding
     errors = warnings = 0
     for name in args.files:
         try:
-            findings = lint(name, labels)
+            findings = lint(name, labels, encoding)
         except (OSError, ValueError) as error:
             print(f'ids-lint: {name}: {error}', file=sys.stderr)
             return 2
-        print(_format_findings(name, findings))
+        # A label can hold a character the console cannot show; print it escaped instead of failing.
+        console = sys.stdout.encoding or 'utf-8'
+        print(_format_findings(name, findings).encode(console, 'backslashreplace').decode(console))
         errors += sum(finding.rule.severity == 'error' for finding in findings)
         warnings += sum(finding.rule.severity == 'warn' for finding in findings)
     print(f'{errors} errors, {warnings} warnings', file=sys.stderr)
diff --git a/src/idsdata/loading.py b/src/idsdata/loading.py
index c1c54ca..f917711 100644
--- a/src/idsdata/loading.py
+++ b/src/idsdata/loading.py
@@ -61,23 +61,47 @@ def _extract_missing(release: Release, directory: Path, selected: list[FileEntry
             check(archives[entry.archive], archive_path)
             checked.add(entry.archive)
         partial = target.with_name(target.name + '.part')
+        member = entry.member or entry.name
         with zipfile.ZipFile(archive_path) as archive:
             try:
-                source = archive.open(entry.name)
+                source = archive.open(member)
             except KeyError:
-                raise ValueError(f'{archive_path} has no member named {entry.name}') from None
+                raise ValueError(f'{archive_path} has no member named {member}') from None
             with source, partial.open('wb') as sink:
                 shutil.copyfileobj(source, sink)
         _ = partial.replace(target)
 
 
+def _drop_blank_rows(table: pa.Table, entry: FileEntry, path: Path) -> pa.Table:
+    """Remove lines that hold no value in any column, after checking their number against the registry."""
+    if table.column('label_raw').null_count:
+        raise ValueError(f'{path} has rows without a label')
+    encoded = table.column('label_raw').combine_chunks().dictionary_encode()
+    seen: list[str] = encoded.dictionary.to_pylist()
+    blank = table.filter(pa.array([raw == '' for raw in seen], pa.bool_()).take(encoded.indices))
+    for column in blank.columns:
+        if column.null_count != len(column) and column.unique().to_pylist() != ['']:
+            raise ValueError(f'{path} has rows with an empty label that hold other values')
+    if blank.num_rows != entry.blank_rows:
+        raise ValueError(f'{path} has {blank.num_rows} lines without values; the registry records {entry.blank_rows}')
+    if not blank.num_rows:
+        return table
+    return table.filter(pa.array([raw != '' for raw in seen], pa.bool_()).take(encoded.indices))
+
+
 def _read_table(release: Release, entry: FileEntry, path: Path) -> pa.Table:
     types = {column.original: _ARROW_TYPES[column.dtype] for column in release.columns}
-    table = pacsv.read_csv(path, convert_options=pacsv.ConvertOptions(column_types=types))
+    table = pacsv.read_csv(
+        path,
+        read_options=pacsv.ReadOptions(encoding=release.encoding),
+        convert_options=pacsv.ConvertOptions(column_types=types),
+    )
     expected = [column.original for column in release.columns]
     if table.column_names != expected:
         raise ValueError(f'{path} does not have the columns recorded for {release.name} {release.version}')
-    table = table.rename_columns([column.name for column in release.columns])
+    table = _drop_blank_rows(table.rename_columns([column.name for column in release.columns]), entry, path)
+    if entry.rows is not None and table.num_rows != entry.rows:
+        raise ValueError(f'{path} has {table.num_rows} rows; the registry records {entry.rows}')
 
     encoded = table.column('label_raw').combine_chunks().dictionary_encode()
     known = {label.raw: label for label in release.labels}
@@ -96,7 +120,11 @@ def _read_table(release: Release, entry: FileEntry, path: Path) -> pa.Table:
 
 def _mapping_key(release: Release) -> str:
     """Changes whenever the recorded columns or labels do, so an old cache is not served after an edit."""
-    return hashlib.sha256(repr((release.columns, release.labels)).encode()).hexdigest()[:8]
+    parts: tuple[object, ...] = (release.columns, release.labels)
+    if release.encoding != 'utf8':
+        # Left out for the default so that caches written before encodings were recorded stay valid.
+        parts = (*parts, release.encoding)
+    return hashlib.sha256(repr(parts).encode()).hexdigest()[:8]
 
 
 def _cached_table(release: Release, entry: FileEntry, directory: Path) -> pa.Table:
@@ -123,7 +151,8 @@ def load(
 
     Columns get their clean names, the source label is kept as ``label_raw`` and
     ``label``, ``is_attack``, ``is_attempted`` and ``source_file`` are added.
-    No row is dropped or relabelled.
+    No flow is dropped or relabelled. Lines that hold no value in any column
+    are left out, and only when the registry records how many a file has.
     """
     release = get_release(name, version)
     directory = data_root(data_dir) / release.name / release.version
diff --git a/src/idsdata/model.py b/src/idsdata/model.py
index 5c600f0..831fac2 100644
--- a/src/idsdata/model.py
+++ b/src/idsdata/model.py
@@ -34,7 +34,12 @@ def bibtex(self) -> str:
 
 @dataclass(frozen=True)
 class FileEntry:
-    """One file of a release. ``archive`` names the archive a table is packed in."""
+    """One file of a release.
+
+    ``archive`` names the archive a table is packed in and ``member`` its path
+    inside that archive when the path is not just ``name``. ``rows`` counts the
+    flows; ``blank_rows`` counts lines that hold no value at all.
+    """
 
     name: str
     sha256: str
@@ -43,6 +48,8 @@ class FileEntry:
     archive: str | None = None
     url: str | None = None
     rows: int | None = None
+    member: str | None = None
+    blank_rows: int = 0
 
 
 @dataclass(frozen=True)
@@ -83,3 +90,5 @@ class Release:
     # Clean name of the column that holds the flow time, and its strptime format.
     time_column: str | None = None
     time_format: str | None = None
+    # Text encoding of the tables, as a Python codec name.
+    encoding: str = 'utf8'
diff --git a/src/idsdata/registry.py b/src/idsdata/registry.py
index b2d9685..95ce017 100644
--- a/src/idsdata/registry.py
+++ b/src/idsdata/registry.py
@@ -9,6 +9,7 @@
 from idsdata._cic_ids2017_engelen2021 import COLUMNS as ENGELEN_COLUMNS
 from idsdata._cic_ids2017_engelen2021 import FILES as ENGELEN_FILES
 from idsdata._cic_ids2017_engelen2021 import LABELS as ENGELEN_LABELS
+from idsdata._cic_ids2017_original import FLOWS_COLUMNS, FLOWS_FILES, FLOWS_LABELS, ML_COLUMNS, ML_FILES, ML_LABELS
 from idsdata.model import Citation, Column, FileEntry, Label, Release
 
 __all__ = ['REGISTRY', 'Citation', 'Column', 'FileEntry', 'Label', 'Release', 'UnknownDatasetError', 'get']
@@ -40,20 +41,62 @@ class UnknownDatasetError(LookupError):
     doi='10.1109/SPW53761.2021.00009',
 )
 
+_ORIGINAL_TERMS = (
+    'No licence is stated. The publisher makes the dataset publicly available for researchers '
+    'and asks users to cite the related paper.'
+)
+_ORIGINAL_NOTES = (
+    'After the request form, the archive is in the CSVs folder. '
+    'The column " Fwd Header Length" appears twice in every file; the second one is named fwd_header_length_1. '
+    'The timestamps are not recorded in one format, so there is no time split for the original versions. '
+)
+
 _RELEASES = (
     Release(
         name='cic-ids2017',
-        version='original',
-        title='CIC-IDS2017',
-        summary='Original release from the Canadian Institute for Cybersecurity, University of New Brunswick.',
+        version='original-flows',
+        title='CIC-IDS2017, labelled flows',
+        summary=(
+            'Original release from the Canadian Institute for Cybersecurity, University of New Brunswick: '
+            'the flows with their ids, addresses and timestamps (GeneratedLabelledFlows.zip).'
+        ),
         homepage='https://www.unb.ca/cic/datasets/ids-2017.html',
         download_page='http://cicresearch.ca/CICDataset/CIC-IDS-2017/',
         access='form',
-        terms=(
-            'No licence is stated. The publisher makes the dataset publicly available for researchers '
-            'and asks users to cite the related paper.'
+        terms=_ORIGINAL_TERMS,
+        citations=(SHARAFALDIN_2018,),
+        files=FLOWS_FILES,
+        columns=FLOWS_COLUMNS,
+        labels=FLOWS_LABELS,
+        encoding='cp1252',
+        retrieved='2026-10-09',
+        notes=_ORIGINAL_NOTES
+        + (
+            'Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv holds 288602 lines without any value; '
+            'load leaves them out. The files are read as Windows-1252, '
+            'which makes the dash in the three web attack labels an en dash. '
+            '21 columns of whole numbers are written with a decimal point in some files and are read as double.'
         ),
+    ),
+    Release(
+        name='cic-ids2017',
+        version='original-ml',
+        title='CIC-IDS2017, machine learning CSVs',
+        summary=(
+            'Original release from the Canadian Institute for Cybersecurity, University of New Brunswick: '
+            'the same flows without ids, addresses and timestamps (MachineLearningCSV.zip).'
+        ),
+        homepage='https://www.unb.ca/cic/datasets/ids-2017.html',
+        download_page='http://cicresearch.ca/CICDataset/CIC-IDS-2017/',
+        access='form',
+        terms=_ORIGINAL_TERMS,
         citations=(SHARAFALDIN_2018,),
+        files=ML_FILES,
+        columns=ML_COLUMNS,
+        labels=ML_LABELS,
+        retrieved='2026-10-09',
+        notes=_ORIGINAL_NOTES
+        + 'The three web attack labels hold the Unicode replacement character (U+FFFD) where the dash was.',
     ),
     Release(
         name='cic-ids2017',
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 3624b53..e426639 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -7,15 +7,19 @@
 
 def test_list(capsys: pytest.CaptureFixture[str]) -> None:
     assert cli.main(['list']) == 0
-    assert capsys.readouterr().out.splitlines() == ['cic-ids2017 original', 'cic-ids2017 engelen2021']
+    assert capsys.readouterr().out.splitlines() == [
+        'cic-ids2017 original-flows',
+        'cic-ids2017 original-ml',
+        'cic-ids2017 engelen2021',
+    ]
 
 
 def test_info(capsys: pytest.CaptureFixture[str]) -> None:
-    assert cli.main(['info', 'cic-ids2017', 'original']) == 0
+    assert cli.main(['info', 'cic-ids2017', 'original-flows']) == 0
     out = capsys.readouterr().out
     assert 'https://www.unb.ca/cic/datasets/ids-2017.html' in out
     assert 'request form' in out
-    assert 'none recorded yet' in out
+    assert 'GeneratedLabelledFlows.zip' in out
 
 
 def test_cite_matches_the_python_api(capsys: pytest.CaptureFixture[str]) -> None:
diff --git a/tests/test_downloading.py b/tests/test_downloading.py
index 84baa60..1832f63 100644
--- a/tests/test_downloading.py
+++ b/tests/test_downloading.py
@@ -87,8 +87,9 @@ def test_download_refuses_releases_it_may_not_fetch(data_dir: Path, host: FakeHo
     assert not data_dir.exists()
 
 
-def test_the_original_release_is_never_fetched() -> None:
-    assert downloading.downloadable(idsdata.info('cic-ids2017', 'original')) == ()
+@pytest.mark.parametrize('version', ['original-flows', 'original-ml'])
+def test_the_original_versions_are_never_fetched(version: str) -> None:
+    assert downloading.downloadable(idsdata.info('cic-ids2017', version)) == ()
 
 
 def test_recorded_download_urls_are_https() -> None:
diff --git a/tests/test_lint.py b/tests/test_lint.py
index 8f93b0e..26812b7 100644
--- a/tests/test_lint.py
+++ b/tests/test_lint.py
@@ -75,6 +75,13 @@ def test_labels_are_checked_against_a_release_when_given(tmp_path: Path) -> None
     assert found(path, ['BENIGN']) == {'label-strings': ["label strings the release does not define: 'Probe'"]}
 
 
+def test_labels_are_read_with_the_given_encoding(tmp_path: Path) -> None:
+    path = write(tmp_path, CLEAN.replace(b'Probe', b'Web \x96 XSS'))
+    labels = ['BENIGN', 'Web \u2013 XSS']
+    assert lint(path, labels, 'cp1252') == []
+    assert found(path, labels) == {'label-strings': ["label strings the release does not define: 'Web \ufffd XSS'"]}
+
+
 def test_a_file_without_a_label_column(tmp_path: Path) -> None:
     content = b'Flow Duration,Rate/s\n1,1.5\n2,2.5\n'
     assert found(write(tmp_path, content)) == {'label-strings': ['the file has no column named Label']}
diff --git a/tests/test_loading.py b/tests/test_loading.py
index 9483f39..527b149 100644
--- a/tests/test_loading.py
+++ b/tests/test_loading.py
@@ -169,8 +169,17 @@ def test_recorded_releases_are_consistent() -> None:
         clean = [column.name for column in release.columns]
         assert len(set(clean)) == len(clean)
         assert clean[-1] == 'label_raw'
+        repeats: dict[str, int] = {}
         for column in release.columns[:-1]:
-            assert column.name == re.sub(r'[^0-9a-z]+', '_', column.original.strip().lower()).strip('_')
+            # A column name that repeats in the source gets a number, like pandas gives it.
+            base = re.sub(r'[^0-9a-z]+', '_', column.original.strip().lower()).strip('_')
+            seen = repeats.get(base, 0)
+            assert column.name == (f'{base}_{seen}' if seen else base)
+            repeats[base] = seen + 1
+        types = {column.original: column.dtype for column in release.columns}
+        assert all(types[column.original] == column.dtype for column in release.columns)
+        for entry in release.files:
+            assert entry.member is None or entry.member.endswith('/' + entry.name)
         raw = [label.raw for label in release.labels]
         assert len(set(raw)) == len(raw)
         assert not any(label.is_attack and label.is_attempted for label in release.labels)
diff --git a/tests/test_original.py b/tests/test_original.py
new file mode 100644
index 0000000..2889f16
--- /dev/null
+++ b/tests/test_original.py
@@ -0,0 +1,135 @@
+"""What the loader needs for files shaped like the original CIC-IDS2017 release."""
+
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+import idsdata
+
+from idsdata import loading
+from idsdata.model import Column, FileEntry, Label
+from tests.conftest import make_archive, make_release, register
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from idsdata.model import Release
+
+# Everything below is made up for the tests. The header repeats a name, the
+# names start with a space, one label holds byte 0x96 and two lines are blank.
+TABLE = b'Flow ID, Size, Rate, Size, Label\r\na,1,1.0,1,BENIGN\r\nb,2,2.0,2,Web Attack \x96 XSS\r\n,,,,\r\n,,,,\r\n'
+COLUMNS = (
+    Column('Flow ID', 'flow_id', 'string'),
+    Column(' Size', 'size', 'int64'),
+    Column(' Rate', 'rate', 'double'),
+    Column(' Size', 'size_1', 'int64'),
+    Column(' Label', 'label_raw', 'string'),
+)
+LABELS = (
+    Label('BENIGN', 'benign', is_attack=False),
+    Label('Web Attack \u2013 XSS', 'web_attack_xss', is_attack=True),
+)
+MEMBER = 'Some Folder /day.csv'
+
+
+def shaped(table: bytes = TABLE, *, rows: int | None = 2, blank_rows: int = 2, encoding: str = 'cp1252') -> Release:
+    archive = make_archive({MEMBER: table})
+    files = (
+        FileEntry('archive.zip', hashlib.sha256(archive).hexdigest(), len(archive), kind='archive'),
+        FileEntry(
+            'day.csv',
+            hashlib.sha256(table).hexdigest(),
+            len(table),
+            archive='archive.zip',
+            member=MEMBER,
+            rows=rows,
+            blank_rows=blank_rows,
+        ),
+    )
+    release = make_release('v1', {'archive.zip': archive}, form=True)
+    return dataclasses.replace(release, files=files, columns=COLUMNS, labels=LABELS, encoding=encoding)
+
+
+def place_archive(data_dir: Path, table: bytes = TABLE) -> None:
+    directory = data_dir / 'synthetic' / 'v1'
+    directory.mkdir(parents=True)
+    _ = (directory / 'archive.zip').write_bytes(make_archive({MEMBER: table}))
+
+
+def test_load_reads_a_table_from_a_folder_inside_the_archive(data_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    register(monkeypatch, shaped())
+    place_archive(data_dir)
+    flows = idsdata.load('synthetic', 'v1', data_dir)
+    assert (data_dir / 'synthetic' / 'v1' / 'day.csv').read_bytes() == TABLE
+    assert flows['source_file'].tolist() == ['day.csv', 'day.csv']
+
+
+def test_load_keeps_both_columns_that_share_a_name(data_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    register(monkeypatch, shaped())
+    place_archive(data_dir)
+    flows = idsdata.load('synthetic', 'v1', data_dir)
+    assert list(flows.columns[:5]) == ['flow_id', 'size', 'rate', 'size_1', 'label_raw']
+    assert flows['size'].tolist() == [1, 2]
+    assert flows['size_1'].tolist() == [1, 2]
+    assert str(flows['size'].dtype) == 'int64'
+
+
+def test_load_decodes_with_the_recorded_encoding(data_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    register(monkeypatch, shaped())
+    place_archive(data_dir)
+    flows = idsdata.load('synthetic', 'v1', data_dir)
+    assert flows['label_raw'].tolist() == ['BENIGN', 'Web Attack \u2013 XSS']
+    assert flows['label'].tolist() == ['benign', 'web_attack_xss']
+
+
+def test_load_leaves_out_the_recorded_blank_lines(data_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    register(monkeypatch, shaped())
+    place_archive(data_dir)
+    assert len(idsdata.load('synthetic', 'v1', data_dir)) == 2
+
+
+def test_load_rejects_blank_lines_the_registry_does_not_record(data_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    register(monkeypatch, shaped(blank_rows=0))
+    place_archive(data_dir)
+    with pytest.raises(ValueError, match='has 2 lines without values; the registry records 0'):
+        _ = idsdata.load('synthetic', 'v1', data_dir)
+
+
+def test_load_rejects_an_empty_label_on_a_line_with_values(data_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    table = TABLE.replace(b',,,,\r\n,,,,\r\n', b',,,,\r\nc,3,3.0,3,\r\n')
+    register(monkeypatch, shaped(table))
+    place_archive(data_dir, table)
+    with pytest.raises(ValueError, match='rows with an empty label that hold other values'):
+        _ = idsdata.load('synthetic', 'v1', data_dir)
+
+
+def test_load_rejects_a_row_count_the_registry_does_not_record(data_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    register(monkeypatch, shaped(rows=3))
+    place_archive(data_dir)
+    with pytest.raises(ValueError, match='has 2 rows; the registry records 3'):
+        _ = idsdata.load('synthetic', 'v1', data_dir)
+
+
+def test_the_original_versions_record_the_same_flows() -> None:
+    flows = idsdata.info('cic-ids2017', 'original-flows')
+    ml = idsdata.info('cic-ids2017', 'original-ml')
+    rows = {}
+    for release in (flows, ml):
+        tables = [entry for entry in release.files if entry.kind == 'table']
+        assert len(tables) == 8
+        assert all(entry.archive and entry.member for entry in tables)
+        rows[release.version] = {entry.name: entry.rows for entry in tables}
+        assert release.time_column is None
+    assert rows['original-flows'] == rows['original-ml']
+    assert sum(count or 0 for count in rows['original-ml'].values()) == 2830743
+    assert {label.label for label in flows.labels} == {label.label for label in ml.labels}
+
+
+def test_the_cache_name_depends_on_a_recorded_encoding() -> None:
+    release = shaped()
+    assert loading._mapping_key(release) != loading._mapping_key(dataclasses.replace(release, encoding='utf8'))  # pyright: ignore[reportPrivateUsage]
diff --git a/tests/test_registry.py b/tests/test_registry.py
index 94b560f..8a2e606 100644
--- a/tests/test_registry.py
+++ b/tests/test_registry.py
@@ -6,11 +6,11 @@
 
 from idsdata import registry
 
-RELEASES = [('cic-ids2017', 'original'), ('cic-ids2017', 'engelen2021')]
+RELEASES = [('cic-ids2017', 'original-flows'), ('cic-ids2017', 'original-ml'), ('cic-ids2017', 'engelen2021')]
 
 
-def test_datasets_lists_both_versions() -> None:
-    assert idsdata.datasets() == {'cic-ids2017': ('original', 'engelen2021')}
+def test_datasets_lists_every_version() -> None:
+    assert idsdata.datasets() == {'cic-ids2017': ('original-flows', 'original-ml', 'engelen2021')}
 
 
 @pytest.mark.parametrize(['name', 'version'], RELEASES)
@@ -24,7 +24,8 @@ def test_info_returns_matching_release(name: str, version: str) -> None:
 
 
 def test_access_modes() -> None:
-    assert idsdata.info('cic-ids2017', 'original').access == 'form'
+    assert idsdata.info('cic-ids2017', 'original-flows').access == 'form'
+    assert idsdata.info('cic-ids2017', 'original-ml').access == 'form'
     assert idsdata.info('cic-ids2017', 'engelen2021').access == 'direct'
 
 
@@ -41,12 +42,12 @@ def test_unknown_dataset_names_the_known_ones() -> None:
 
 
 def test_unknown_version_names_the_known_ones() -> None:
-    with pytest.raises(idsdata.UnknownDatasetError, match='known versions: original, engelen2021'):
+    with pytest.raises(idsdata.UnknownDatasetError, match='known versions: original-flows, original-ml, engelen2021'):
         _ = idsdata.info('cic-ids2017', 'liu2022')
 
 
 def test_cite_original() -> None:
-    bibtex = idsdata.cite('cic-ids2017', 'original')
+    bibtex = idsdata.cite('cic-ids2017', 'original-ml')
     assert bibtex.startswith('@inproceedings{sharafaldin2018toward,')
     assert '10.5220/0006639801080116' in bibtex
     assert 'pages     = {108--116},' in bibtex