text:
FROM python:3.9-slim-buster

LABEL name="Dockerized Endpoint Testing"

WORKDIR /app
COPY docker/reqs.txt ./
COPY src ./src
COPY docker/entrypoint.sh ./

RUN pip3 --version
RUN python3 --version
RUN pip3 install -r reqs.txt

ENTRYPOINT ["/app/entrypoint.sh"]


text:
[toolchain]
# The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy.
# https://rust-lang.github.io/rustup/concepts/profiles.html
profile = "default"
channel = "1.66.0"

repo_name: sstadick/hck

repo_language: Rust

mime_type: text/plain


text:
# Changelog

## v0.8.0

- [Bugfix](https://github.com/sstadick/hck/issues/59) Fixed output column reordering
- Updated deps and added dependabot

## v0.7.5

- [Bugfix] Invalid thirdparty file fixed

## v0.7.4

- [Improvement] Updated deps to get latest gzp and thereby latest flate2, fix thirdparty.yml

## v0.7.3

- [Improvement] Updated deps to get latest gzp and thereby latest flate2

## v0.7.2

- [Improvement](https://github.com/sstadick/hck/pull/51) Update from structopt to clap v3
- [Improvement] Updated deps

## v0.7.1

- bugfix Apply unescaping to headers when selecting via `-F`

## v0.7.0

- [Improvement](https://github.com/sstadick/hck/issues/46#issuecomment-974189408) Allow for passing escaped sequences in for the input and output delimiteres. i.e '\t' or '\n' can be passed in directly instead of needing to add $'\n' on the command line.
- [Improvement](https://github.com/sstadick/hck/issues/46#issuecomment-974189408) Add a flag `-I` / `--use-input-delim` to reuse the input delimiter if it is a literal as the output delimiter. An error will be raised if `-L` is not specified or if `-D` is specified.

## v0.6.7

- [Bugfix](https://github.com/sstadick/hck/issues/44) allow arguments of options to start with a "-".

## v0.6.6

- Performance improvements. On literal byte separators `hck` is now faster than it previously was and is much faster on inputs that only use the first few fields in a line.

## v0.6.5

- [Bugfix](https://github.com/sstadick/hck/issues/38) for files using fast-path code that have empty first lines

## v0.6.4

- Added thirdparty license in prep for conda-forge

## v0.6.3

- Change the number of compression threads used by default
- Update gzp version

## v0.6.2

- Output BGZF output by default, which can be indexed and queried with tabix
- Read input gzipped files a `MultiGzDecoder` which is more flexible

## v0.6.1

- PGO build fix

## v0.6.0

- Speed up edge case where columns may have already been consumed i.e. `-f1-3,5,2-`
- Remove pigz binary from compression search
- Add native gz decompression
- Add native gz output compression via `gzp`

## v0.5.4

- [Bugfix](https://github.com/sstadick/hck/issues/30) Better handling of duplicate selected fields, fixed output ordering when duplicate fields were selected. Added clarification to README regarding mixing by-index and by-header field selction / reordering.

## v0.5.3

- Bugfix, allow headers specified to be excluded to not be found

## v0.5.2

- [PR24](https://github.com/sstadick/hck/pull/24) Removed the now defunct profile guided optimization shell scripts and all references to them in favor of the `justfile` that was added in `v0.5.0`
- [Bugfix](https://github.com/sstadick/hck/issues/26) fixes incorrect handling of header line for non-stdin inputs, fixes incorrect parsing of last header fields (now strips newline before matching), fixes option parsing so that the `-F` and `-E` options wont' try to consume the positional input arguments. Huge thanks to @learnbyexample for their detailed bug report.
- Change: An error will now be raised when a specified header is not found. This differs from the convention used by the selecion-by-index, which tries to match `cut`. The reasoning is that it is generally harder to type out each header field and if a header is not found you want to know about it.

## v0.5.1

- Fix the version in the binary to match the actual version

## v0.5.0

- Added `--exclude|-e` flag to select a set of fields to exclude. These fields may overlap with the `-f` flag and take precedence over fields selected by `-f`.
- Added `--exclude-header|-E` flag to select a set of headers to exclude. These may mix and match with `-e` `-f` and `F`. The `-r` flag will cause the headers to be treated as a regex.
- As part of the `-e` additional, the default behavior if now headers or fields are specified is to assume `-f1-`, which allow the user to do `hck -e 3,8,290`.
- pigz is now a supported decompression binary, if it's not present `hck` defaults back to `gzip`.
- Decided against adding a greedy heuristic because it actually had worse performance on the most common case of `\s` (but better on `[[:space:]]`, which was odd).
  - The place where this would make sense would be searching a literal space character greedily (like awk), but that kind of goes against the way the delimiters are documented to work
  - It may be worth adding that special case at some point?
- Moves CI to using justfile instead of pgo scripts.
- Fixes several issues in benchmarks
  - `choose` was not using fastest path and had the wrong input file
  - All tools splitting by a space regex were incorrectly parsing the header line in the multichar data, header line is now fixed
- Fixes bug with reordering of only two fields

## v0.4.2

- Fixed deb CI


text:
llvm_profdata := `find $(rustc --print sysroot) -name llvm-profdata -type f`

pgo-data := join(justfile_directory(), "pgo-data")
data := join(pgo-data, "data.csv")
spaced-data := join(pgo-data, "spaced_data.txt")
gzip-data := join(pgo-data, "data.csv.gz")

prep:
    rustup component add llvm-tools-preview
    mkdir -p {{pgo-data}}

clean:
    rm -rf {{pgo-data}}

get-data: prep
    wget -q -S -O - https://archive.ics.uci.edu/ml/machine-learning-databases/00347/all_train.csv.gz \
        | gunzip \
        | head -n 1000000 \
        > {{pgo-data}}/data.csv

generate-pgo-data: prep get-data
    RUSTFLAGS="-Cprofile-generate={{pgo-data}}" \
        cargo build --release --locked

run-instrumented-binary: generate-pgo-data
    # single-byte in order
    {{justfile_directory()}}/target/release/hck -Ld, -f1,8,19 -e 8 "{{data}}"  > /dev/null
    {{justfile_directory()}}/target/release/hck -Ld, --no-mmap -f1,8,19 -e 8 "{{data}}"  > /dev/null

    # single byte reorder
    {{justfile_directory()}}/target/release/hck -Ld, -f1,19,8 -e 8 "{{data}}" > /dev/null
    {{justfile_directory()}}/target/release/hck -Ld, --no-mmap -f1,19,8 -e 8 "{{data}}" > /dev/null

    # single byte regex
    {{justfile_directory()}}/target/release/hck -d, -f1,8,19 -e 8 "{{data}}" > /dev/null
    {{justfile_directory()}}/target/release/hck -d, -f1,8,19 --no-mmap -e 8 "{{data}}" > /dev/null

    # make multi-space file
    {{justfile_directory()}}/target/release/hck -Ld, -D '   ' -f1,8,19 -e 8 "{{data}}" > "{{spaced-data}}"

    # multi byte regex
    {{justfile_directory()}}/target/release/hck -d '\s+' -f1,8,19 -e 8 "{{spaced-data}}" > /dev/null
    {{justfile_directory()}}/target/release/hck -d '\s+' --no-mmap -f1,8,19 -e 8 "{{spaced-data}}" > /dev/null
    {{justfile_directory()}}/target/release/hck -d '[[:space:]]+' -f1,8,19 -e 8 "{{spaced-data}}" > /dev/null
    {{justfile_directory()}}/target/release/hck -d '[[:space:]]+' --no-mmap -f1,8,19 -e 8 "{{spaced-data}}" > /dev/null

    # gzip decoding
    gzip -c "{{data}}" > "{{gzip-data}}"
    {{justfile_directory()}}/target/release/hck -Z -Ld, -D, -f1- -e 8 "{{data}}" > "{{gzip-data}}"
    {{justfile_directory()}}/target/release/hck -z -Ld, -f1,19,8 -e 8 "{{gzip-data}}" > /dev/null

clean-data: run-instrumented-binary
    rm {{data}}
    rm {{spaced-data}}
    rm {{gzip-data}}

merge-prof-data llvm-profdata=`find ~/.rustup/toolchains/*/lib/rustlib/*/bin/ -name llvm-profdata -type f`: clean-data prep
    @echo "Using llvm-profdata: {{llvm-profdata}}"
    {{llvm-profdata}} merge -o {{pgo-data}}/merged.profdata {{pgo-data}}

build: merge-prof-data
    RUSTFLAGS="-Cllvm-args=-pgo-warn-missing-function -Cprofile-use={{pgo-data}}/merged.profdata" \
        cargo build --release --locked

build-native: merge-prof-data
    RUSTFLAGS="-Ctarget-cpu=native -Cllvm-args=-pgo-warn-missing-function -Cprofile-use={{pgo-data}}/merged.profdata" \
        cargo build --release --locked

install-native: build-native
    cp {{justfile_directory()}}/target/release/hck ~/.cargo/bin/hck


text:
[package]
edition = "2018"
name = "hck"
version = "0.9.2"
authors = ["Seth Stadick <sstadick@gmail.com>"]
license = "Unlicense OR MIT"
repository = "https://github.com/sstadick/hck"
description = "A sharp cut(1) clone."
documentation = "https://docs.rs/hck"
readme = "README.md"
categories = ["text-processing", "command-line-interface"]
keywords = ["text", "cli"]

[lib]
name = "hcklib"
path = "src/lib/mod.rs"

[[bin]]
name = "hck"
path = "src/main.rs"

[profile.release]
lto = "fat"
codegen-units = 1

[dependencies]
anyhow = "1.0.70"
bstr = "1.4.0"
clap = { version = "4.2.1", features = ["derive"] }
env_logger = "0.10.0"
flate2 = { version = "1.0.25", features = ["zlib-ng-compat"], default-features = false }
gzp = { version = "0.11.3", default-features = false, features = ["deflate_zlib_ng","libdeflate"] }
grep-cli = "0.1.7"
lazy_static = "1.4.0"
log = "0.4.17"
memmap = { package = "memmap2", version = "0.5.10" }
memchr = "2.5.0"
num_cpus = "1.15.0"
regex = "1.7.3"
ripline = "0.1.0"
termcolor = "1.2.0"
thiserror = "1.0.40"
git-version = "0.3.5"

[dev-dependencies]
tempfile = "3.5.0"
rstest = "0.17.0"


text:
#!/usr/bin/bash
set -eo pipefail

test_data="$1"
cp "${test_data}" ./hyper_data.txt

# Single Character Delimiter tests

hyperfine --warmup 3 -m 5 --export-markdown single_char.md --show-output \
    'hck -Ld, -f1,8,19 ./hyper_data.txt > /dev/null' \
    'hck -Ld, -f1,8,19 --no-mmap ./hyper_data.txt > /dev/null' \
    'hck -d, -f1,8,19  ./hyper_data.txt > /dev/null' \
    'hck -d, -f1,8,19  --no-mmap ./hyper_data.txt > /dev/null' \
    "tsv-select -d, -f 1,8,19 ./hyper_data.txt > /dev/null" \
    'choose -f ',' -i ./hyper_data.txt 0 7 18  > /dev/null' \
    "xsv select -d, 1,8,19 ./hyper_data.txt > /dev/null" \
    "awk -F, '{print \$1, \$8, \$19}' ./hyper_data.txt > /dev/null" \
    'cut -d, -f1,8,19 ./hyper_data.txt > /dev/null'

# Multi character delimiter tests, create a file with three spaces as delmiter
hck -d, -f1- -D '   ' -o ./hyper_data_multichar.txt ./hyper_data.txt
sed -i 's/# label/#label/' ./hyper_data_multichar.txt

# Notes:
#   - xsv gets messed up by the first commented line in the section, so tail skips it, the point is to run these commands as we would in the wild
hyperfine --warmup 3 -m 5 --export-markdown multi_char.md --show-output \
    "hck -Ld'   ' -f1,8,19 ./hyper_data_multichar.txt > /dev/null" \
    "hck -Ld'   ' -f1,8,19 --no-mmap ./hyper_data_multichar.txt > /dev/null" \
    "hck -d'   ' -f1,8,19 ./hyper_data_multichar.txt > /dev/null" \
    "hck -d'   ' --no-mmap -f1,8,19 ./hyper_data_multichar.txt > /dev/null" \
    "hck -d'[[:space:]]+' -f1,8,19 ./hyper_data_multichar.txt > /dev/null" \
    "hck -d'[[:space:]]+' --no-mmap -f1,8,19 ./hyper_data_multichar.txt > /dev/null" \
    "hck -d'\s+' -f1,8,19 ./hyper_data_multichar.txt > /dev/null" \
    "hck -d'\s+' -f1,8,19 --no-mmap ./hyper_data_multichar.txt > /dev/null" \
    "choose -f '   ' -i ./hyper_data_multichar.txt 0 7 18  > /dev/null" \
    "choose -f '[[:space:]]' -i ./hyper_data_multichar.txt 0 7 18  > /dev/null" \
    "choose -f '\s' -i ./hyper_data_multichar.txt 0 7 18  > /dev/null" \
    "awk -F' ' '{print \$1, \$8 \$19}' ./hyper_data_multichar.txt > /dev/null" \
    "awk -F'   ' '{print \$1, \$8, \$19}' ./hyper_data_multichar.txt > /dev/null" \
    "awk -F'[:space:]+' '{print \$1, \$8, \$19}' ./hyper_data_multichar.txt > /dev/null" \
    "< ./hyper_data_multichar.txt tr -s ' ' | cut -d ' ' -f1,8,19 > /dev/null" \
    "< ./hyper_data_multichar.txt tr -s ' ' | xsv select -d ' ' 1,8,19 --no-headers > /dev/null" \
    "< ./hyper_data_multichar.txt tr -s ' ' | hck -Ld' ' -f1,8,19 > /dev/null" \
    "< ./hyper_data_multichar.txt tr -s ' ' | tsv-select -d ' ' -f 1,8,19 > /dev/null"

# rm ./hyper_data.txt
# rm ./hyper_data_multichar.txt


text:
# 🪓 hck

  A sharp cut(1) clone.

_`hck` is a shortening of `hack`, a rougher form of `cut`._

A close to drop in replacement for cut that can use a regex delimiter instead of a fixed string.
Additionally this tool allows for specification of the order of the output columns using the same column selection syntax as cut (see below for examples).

No single feature of `hck` on its own makes it stand out over `awk`, `cut`, `xsv` or other such tools. Where `hck` excels is making common things easy, such as reordering output fields, or splitting records on a weird delimiter.
It is meant to be simple and easy to use while exploring datasets.
Think of this as filling a gap between `cut` and `awk`.

`hck` is dual-licensed under MIT or the [UNLICENSE](https://unlicense.org/).

## Features

- Reordering of output columns! i.e. if you use `-f4,2,8` the output columns will appear in the order `4`, `2`, `8`
- Delimiter treated as a regex, i.e. you can split on multiple spaces without and extra pipe to `tr`!
- Specification of output delimiter
- Selection of columns by header string literal with the `-F` option, or by regex by setting the `-r` flag
- Input files will be automatically decompressed if their file extension is recognizable and a local binary exists to perform the decompression (similar to ripgrep). See [Decompression](#decompression).
- Output can be gzip compressed using the multi-threaded compressors from [`gzp`](https://github.com/sstadick/gzp) with `-Z` flag
  - This gzipped output is in BGZF format and can be indexed and queried with `tabix`
- Exclude fields by index or by header.
- Speed

## Non-goals

- `hck` does not aim to be a complete CSV / TSV parser a la `xsv` which will respect quoting rules. It acts similar to `cut` in that it will split on the delimiter no matter where in the line it is.
- Delimiters cannot contain newlines... well they can, they will just never be seen. `hck` will always be a line-by-line tool where newlines are the standard `\n` `\r\n`.

## Install

- Homebrew / Linuxbrew

```bash
brew tap sstadick/hck
brew install hck
```

- Conda

```bash
# Note, this version lags by about a day
conda install -c conda-forge hck
```

- MacPorts

```bash
# Note, version may lag behind latest
sudo port selfupdate
sudo port install hck
```

- Debian (Ubuntu)

```bash
curl -LO https://github.com/sstadick/hck/releases/download/<latest>/hck-linux-amd64.deb
sudo dpkg -i hck-linux-amd64.deb
```

\* Built with profile guided optimizations

- With the Rust toolchain:

```bash
export RUSTFLAGS='-C target-cpu=native'
cargo install hck
```

- From the [releases page](https://github.com/sstadick/hck/releases) (the binaries have been built with profile guided optimizations)

- Or, if you want the absolute fastest possible build that makes use of profile guided optimizations AND native cpu features:

```bash
# Assumes you are on stable rust
# NOTE: this won't work on windows, see CI for linked issue
cargo install just
git clone https://github.com/sstadick/hck
cd hck
just install-native
```

- PRs are both welcome and encouraged for adding more packaging options and build types! I'd especially welcome PRs for the windows family of package managers / general making sure things are windows friendly.

## Packaging status

[![Packaging status](https://repology.org/badge/vertical-allrepos/hck.svg)](https://repology.org/project/hck/versions)

## Examples

### Splitting with a string literal

```bash
❯ hck -Ld' ' -f1-3,5- ./README.md | head -n4
#       🪓      hck

<p      align="center">
                <a      src="https://github.com/sstadick/hck/workflows/Check/badge.svg" alt="Build      Status"></a>
```

### Splitting with a regex delimiter

```bash
# note, '\s+' is the default
❯ ps aux | hck -f1-3,5- | head -n4
USER    PID     %CPU    VSZ     RSS     TTY     STAT    START   TIME    COMMAND
root    1       0.0     169452  13472   ?       Ss      Jun21   0:19    /sbin/init      splash
root    2       0.0     0       0       ?       S       Jun21   0:00    [kthreadd]
root    3       0.0     0       0       ?       I<      Jun21   0:00    [rcu_gp]
```

### Reordering output columns

```bash
❯ ps aux | hck -f2,1,3- | head -n4
PID     USER    %CPU    %MEM    VSZ     RSS     TTY     STAT    START   TIME    COMMAND
1       root    0.0     0.0     169452  13472   ?       Ss      Jun21   0:19    /sbin/init      splash
2       root    0.0     0.0     0       0       ?       S       Jun21   0:00    [kthreadd]
3       root    0.0     0.0     0       0       ?       I<      Jun21   0:00    [rcu_gp]
```

### Excluding output columns

```bash
❯ ps aux | hck -e3,5 | head -n4
USER    PID     %MEM    RSS     TTY     STAT    START   TIME    COMMAND
root    1       0.0     14408   ?       Ss      Jun21   0:27    /sbin/init      splash
root    2       0.0     0       ?       S       Jun21   0:01    [kthreadd]
root    3       0.0     0       ?       I<      Jun21   0:00    [rcu_gp]
```

### Excluding output columns by header regex

```bash
❯  ps aux | hck -r -E "CPU" -E "^ST.*" | head -n4
USER    PID     %MEM    VSZ     RSS     TTY     TIME    COMMAND
root    1       0.0     170224  14408   ?       0:27    /sbin/init      splash
root    2       0.0     0       0       ?       0:01    [kthreadd]
root    3       0.0     0       0       ?       0:00    [rcu_gp]
```

### Changing the output record separator

```bash
❯ ps aux | hck -D'___' -f2,1,3 | head -n4
PID___USER___%CPU
1___root___0.0
2___root___0.0
3___root___0.0
```

### Select columns with regex

```bash
# Note the order match the order of the -F args
ps aux | hck -r -F '^ST.*' -F '^USER$' | head -n4
STAT    START   USER
Ss      Jun21   root
S       Jun21   root
I<      Jun21   root
```

### Automagic decompresion

```bash
❯ gzip ./README.md
❯ hck -Ld' ' -f1-3,5- -z ./README.md.gz | head -n4
#       🪓      hck

<p      align="center">
                <a      src="https://github.com/sstadick/hck/workflows/Check/badge.svg" alt="Build      Status"></a>
```

### Splitting on multiple characters

```bash
# with string literal
❯ printf 'this$;$is$;$a$;$test\na$;$b$;$3$;$four\n' > test.txt
❯ hck -Ld'$;$' -f3,4 ./test.txt
a       test
3       four
# with an interesting regex
❯ printf 'this123__is456--a789-test\na129_-b849-_3109_-four\n' > test.txt
❯ hck -d'\d{3}[-_]+' -f3,4 ./test.txt
a       test
3       four
```

### Splitting by-index and by-header

This one requires some explaining first. Basically, by-index and by-header selections each have their own "order", and then the orders are merged ex:

```bash
❯ printf 'a,b,c,d,e\n1,2,3,4,5\n' | hck -d, -D: -f3 -F 'b' -F 'a'
b:c:a
2:3:1
```

In the by-index group, we've specified column 3 to be in output position 0. In the by-header group, we've specified that column `b` be in position 0. They by-index and by-header selections are merged together and when merging, if there are two outputs specified to be in the same output position the are output in input order (input meaning the order of columns in the input data).

This can lead to unexpected outcomes, such as the following example where `a` now comes first in the output when compared to the example above.

```bash
❯ printf 'a,b,c,d,e\n1,2,3,4,5\n' | hck -d, -D: -f3 -F 'a'
a:c
1:3
```

Takeaway: be careful when a specific output order is desired and you are mixing and matching by-index and by-header field selections.

## Benchmarks

This set of benchmarks is simply meant to show that `hck` is in the same ballpark as other tools. These are meant to capture real world usage of the tools, so in the multi-space delimiter benchmark for `gcut`, for example, we use `tr` to convert the space runs to a single space and then pipe to `gcut`.

*Note* this is not meant to be an authoritative set of benchmarks, it is just meant to give a relative sense of performance of different ways of accomplishing the same tasks.

#### Hardware

Ubuntu 20 AMD Ryzen 9 3950X 16-Core Processor w/ 64 GB DDR4 memory and 1TB NVMe Drive

#### Data

The [all_train.csv](https://archive.ics.uci.edu/ml/machine-learning-databases/00347/all_train.csv.gz) data is used.

This is a CSV dataset with 7 million lines. We test it both using `,` as the delimiter, and then also using `\s\s\s` as a delimiter.

PRs are welcome for benchmarks with more tools, or improved (but still realistic) pipelines for commands.

#### Tools

`cut`:
  - https://www.gnu.org/software/coreutils/manual/html_node/The-cut-command.html
  - 8.30

`mawk`:
  - https://invisible-island.net/mawk/mawk.html
  - v1.3.4

`xsv`:
  - https://github.com/BurntSushi/xsv
  - v0.13.0 (compiled locally with optimizations)

`tsv-utils`:
  - https://github.com/eBay/tsv-utils
  - v2.2.0 (ldc2, compiled locally with optimizations)

`choose`:
  - https://github.com/theryangeary/choose
  - v1.3.2 (compiled locally with optimizations)

### Single character delimiter benchmark

| Command                                                      |      Mean [s] | Min [s] | Max [s] |    Relative |
| :----------------------------------------------------------- | ------------: | ------: | ------: | ----------: |
| `hck -Ld, -f1,8,19 ./hyper_data.txt > /dev/null`             | 1.198 ± 0.015 |   1.185 |   1.215 |        1.00 |
| `hck -Ld, -f1,8,19 --no-mmap ./hyper_data.txt > /dev/null`   | 1.349 ± 0.029 |   1.320 |   1.389 | 1.13 ± 0.03 |
| `hck -d, -f1,8,19  ./hyper_data.txt > /dev/null`             | 1.649 ± 0.023 |   1.624 |   1.673 | 1.38 ± 0.03 |
| `hck -d, -f1,8,19  --no-mmap ./hyper_data.txt > /dev/null`   | 1.869 ± 0.019 |   1.842 |   1.894 | 1.56 ± 0.02 |
| `tsv-select -d, -f 1,8,19 ./hyper_data.txt > /dev/null`      | 1.702 ± 0.021 |   1.687 |   1.734 | 1.42 ± 0.02 |
| `choose -f , -i ./hyper_data.txt 0 7 18  > /dev/null`        | 4.285 ± 0.092 |   4.214 |   4.428 | 3.58 ± 0.09 |
| `xsv select -d, 1,8,19 ./hyper_data.txt > /dev/null`         | 5.693 ± 0.042 |   5.635 |   5.745 | 4.75 ± 0.07 |
| `awk -F, '{print $1, $8, $19}' ./hyper_data.txt > /dev/null` | 4.993 ± 0.029 |   4.959 |   5.030 | 4.17 ± 0.06 |
| `cut -d, -f1,8,19 ./hyper_data.txt > /dev/null`              | 7.541 ± 1.250 |   6.827 |   9.769 | 6.30 ± 1.05 |

### Multi-character delimiter benchmark

| Command                                                                                |       Mean [s] | Min [s] | Max [s] |     Relative |
| :-------------------------------------------------------------------------------------------- | -------------: | ------: | ------: | -----------: |
| `hck -Ld'   ' -f1,8,19 ./hyper_data_multichar.txt > /dev/null`                                |  1.718 ± 0.003 |   1.715 |   1.722 |         1.00 |
| `hck -Ld'   ' -f1,8,19 --no-mmap ./hyper_data_multichar.txt > /dev/null`                      |  2.191 ± 0.072 |   2.135 |   2.291 |  1.28 ± 0.04 |
| `hck -d'   ' -f1,8,19 ./hyper_data_multichar.txt > /dev/null`                                 |  2.180 ± 0.029 |   2.135 |   2.208 |  1.27 ± 0.02 |
| `hck -d'   ' --no-mmap -f1,8,19 ./hyper_data_multichar.txt > /dev/null`                       |  2.542 ± 0.014 |   2.529 |   2.565 |  1.48 ± 0.01 |
| `hck -d'[[:space:]]+' -f1,8,19 ./hyper_data_multichar.txt > /dev/null`                        |  8.597 ± 0.023 |   8.575 |   8.631 |  5.00 ± 0.02 |
| `hck -d'[[:space:]]+' --no-mmap -f1,8,19 ./hyper_data_multichar.txt > /dev/null`              |  8.890 ± 0.013 |   8.871 |   8.903 |  5.17 ± 0.01 |
| `hck -d'\s+' -f1,8,19 ./hyper_data_multichar.txt > /dev/null`                                 | 10.014 ± 0.247 |   9.844 |  10.449 |  5.83 ± 0.14 |
| `hck -d'\s+' -f1,8,19 --no-mmap ./hyper_data_multichar.txt > /dev/null`                       | 10.173 ± 0.035 |  10.111 |  10.193 |  5.92 ± 0.02 |
| `choose -f '   ' -i ./hyper_data_multichar.txt 0 7 18  > /dev/null`                           |  6.537 ± 0.148 |   6.452 |   6.799 |  3.80 ± 0.09 |
| `choose -f '[[:space:]]' -i ./hyper_data_multichar.txt 0 7 18  > /dev/null`                   | 10.656 ± 0.219 |  10.484 |  10.920 |  6.20 ± 0.13 |
| `choose -f '\s' -i ./hyper_data_multichar.txt 0 7 18  > /dev/null`                            | 37.238 ± 0.153 |  37.007 |  37.383 | 21.67 ± 0.10 |
| `awk -F' ' '{print $1, $8 $19}' ./hyper_data_multichar.txt > /dev/null`                       |  6.673 ± 0.064 |   6.595 |   6.734 |  3.88 ± 0.04 |
| `awk -F'   ' '{print $1, $8, $19}' ./hyper_data_multichar.txt > /dev/null`                    |  5.947 ± 0.098 |   5.896 |   6.121 |  3.46 ± 0.06 |
| `awk -F'[:space:]+' '{print $1, $8, $19}' ./hyper_data_multichar.txt > /dev/null`             | 11.080 ± 0.215 |  10.881 |  11.376 |  6.45 ± 0.13 |
| `< ./hyper_data_multichar.txt tr -s ' ' \| cut -d ' ' -f1,8,19 > /dev/null`                   |  7.471 ± 0.066 |   7.397 |   7.561 |  4.35 ± 0.04 |
| `< ./hyper_data_multichar.txt tr -s ' ' \| xsv select -d ' ' 1,8,19 --no-headers > /dev/null` |  6.172 ± 0.068 |   6.071 |   6.235 |  3.59 ± 0.04 |
| `< ./hyper_data_multichar.txt tr -s ' ' \| hck -Ld' ' -f1,8,19 > /dev/null`                   |  6.171 ± 0.112 |   5.975 |   6.243 |  3.59 ± 0.07 |
| `< ./hyper_data_multichar.txt tr -s ' ' \| tsv-select -d ' ' -f 1,8,19 > /dev/null`           |  6.202 ± 0.130 |   5.984 |   6.290 |  3.61 ± 0.08 |

## Decompression

The following table indicates the file extension / binary pairs that are used to try to decompress a file when the `-z` option is specified:

| Extension | Binary                   | Type       |
| :-------- | :----------------------- | :--------- |
| `*.gz`    | Native                   | gzip       |
| `*.tgz`   | `gzip -d -c`             | gzip       |
| `*.bz2`   | `bzip2 -d -c`            | bzip2      |
| `*.tbz2`  | `bzip2 -d -c`            | bzip2      |
| `*.xz`    | `xz -d -c`               | xz         |
| `*.txz`   | `xz -d -c`               | xz         |
| `*.lz4`   | `lz4 -d -c`              | lz4        |
| `*.lzma`  | `xz --format=lzma -d -c` | lzma       |
| `*.br`    | `brotli -d -c`           | brotli     |
| `*.zst`   | `zstd -d -c`             | zstd       |
| `*.zstd`  | `zstd -q -d -c`          | zstd       |
| `*.Z`     | `uncompress -c`          | uncompress |

When a file with one of the extensions above is found, `hck` will open a subprocess running the the decompression tool listed above and read from the output of that tool. If the binary can't be found then `hck` will try to read the compressed file as is. See [`grep_cli`](https://github.com/BurntSushi/ripgrep/blob/9eddb71b8e86a04d7048b920b9b50a2e97068d03/crates/cli/src/decompress.rs#L468) for source code. The end goal is to add a similar preprocessor as [ripgrep](https://github.com/BurntSushi/ripgrep/blob/master/GUIDE.md#preprocessor). Where there are multiple binaries for a given type, they are tried in the order listed above.

## Profile Guided Optimization

See the `pgo*.sh` scripts for how to build this with optimizations. You will need to install the llvm tools via `rustup component add llvm-tools-preview` for this to work. Building with PGO seems to improve performance anywhere from 5-30% depending on the platform and codepath. i.e. on mac os it seems to have a larger effect, and on the regex codepath it also seems to have a greater effect.

## TODO

- Add output compression detection when writing to a file
- Don't reparse fields / headers for each new file
- Figure out how to better reuse / share a vec
- Support indexing from the end (unlikely though)
- Bake in grep / filtering somehow (this will not be done at the expense of the primary utility of `hck`)
- Move tests from main to core
- Add more tests all around
- Experiment with parallel parser as described [here](https://www.semanticscholar.org/paper/Instant-Loading-for-Main-Memory-Databases-M%C3%BChlbauer-R%C3%B6diger/a1b067fc941d6727169ec18a882080fa1f074595?p2df) This should be very doable given we don't care about escaping quotes and such.

## More packages and builds

https://github.com/sharkdp/bat/blob/master/.github/workflows/CICD.yml

## References

- [rust-coreutils-cut](https://github.com/uutils/coreutils/blob/e48ff9dd9ee0d55da285f99d75f6169a5e4e7acc/src/uu/cut/src/cut.rs)
- [ripgrep](https://github.com/BurntSushi/ripgrep/tree/master/crates/searcher)


text:
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates

version: 2
updates:
  - package-ecosystem: "cargo" # See documentation for possible values
    directory: "/" # Location of package manifests
    schedule:
      interval: "weekly"


text:
name: Check

on: [push, pull_request]

env:
  CARGO_TERM_COLOR: always

jobs:
  check:
    name: Check
    runs-on: ubuntu-latest
    steps:
      - name: Checkout sources
        uses: actions/checkout@v2

      - name: Install stable toolchain
        uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          override: false

      - name: Cache dependencies
        uses: Swatinem/rust-cache@v1

      - name: Run cargo check
        uses: actions-rs/cargo@v1
        with:
          command: check

  lints:
    name: Lints
    runs-on: ubuntu-latest
    steps:
      - name: Checkout sources
        uses: actions/checkout@v2

      - name: Install stable toolchain
        uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          override: false
          components: rustfmt, clippy

      - name: Cache dependencies
        uses: Swatinem/rust-cache@v1

      - name: Run cargo fmt
        uses: actions-rs/cargo@v1
        with:
          command: fmt
          args: --all -- --check

      - name: Run cargo clippy
        uses: actions-rs/cargo@v1
        with:
          command: clippy
          args: -- -D warnings

  test:
    name: Test Suite
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, macOS-latest, windows-latest]
    steps:
      - name: Checkout sources
        uses: actions/checkout@v2

      - name: Install stable toolchain
        uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          override: false

      - name: Cache dependencies
        uses: Swatinem/rust-cache@v1

      - name: Run tests
        run: cargo test --verbose

      - name: Install cargo-bundle-licenses
        run: cargo install cargo-bundle-licenses

      - name: Check licenses
        run: cargo bundle-licenses --format yaml --output CI.THIRDPARTY.yml --previous THIRDPARTY.yml --check-previous


text:
name: Publish

on:
  push:
    tags:
      - "*"

jobs:
  publish:
    name: Publish for ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
            artifact_name: hck
            asset_name: hck-linux-amd64
          - os: macos-latest
            artifact_name: hck
            asset_name: hck-macos-amd64
          - os: windows-latest
            artifact_name: hck.exe
            asset_name: hck-windows-amd64.exe

    steps:
      - uses: actions/checkout@v2
      - uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Install LLVM Tools
        run: rustup component add llvm-tools-preview

      - name: Build
        shell: bash
        run: |
          if [ "${{ matrix.os }}" = "windows-latest" ]; then
            # Windows MSVC does not yet support PGO
            # https://github.com/rust-lang/rust/issues/61002
            cargo build --release --locked
          else
            just build
          fi

      - name: Build archive
        shell: bash
        run: |
          staging="${{matrix.asset_name}}-src"
          mkdir -p "$staging"

          cp {README.md,UNLICENSE,LICENSE-MIT} "$staging/"
          cp {Cargo.toml,Cargo.lock} "$staging/"
          cp justfile "$staging/"
          cp -R ./src "./$staging/src"

          if [ "${{ matrix.os }}" = "windows-latest" ]; then
            7z a "${staging}.zip" "$staging"
            echo "ASSET=${staging}.zip" >> $GITHUB_ENV
          else
            mkdir -p "${staging}/pgo-data"
            cp ./pgo-data/merged.profdata "./${staging}/pgo-data/merged.profdata"
            tar czf "${staging}.tar.gz" "${staging}"
            echo "ASSET=${staging}.tar.gz" >> $GITHUB_ENV
          fi

      - name: Create deb artifact
        shell: bash
        run: |
          if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
            cargo install cargo-deb
            asset_path="${{ matrix.asset_name }}.deb"
            RUSTFLAGS="-Cllvm-args=-pgo-warn-missing-function -Cprofile-use=$(pwd)/pgo-data/merged.profdata" cargo deb --output ./"${asset_path}"
            echo "DEB=${asset_path}" >> $GITHUB_ENV
          fi

      - name: Upload deb package
        uses: svenstaro/upload-release-action@v2
        if: matrix.os == 'ubuntu-latest'
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: ${{ env.DEB }}
          asset_name: ${{ env.DEB }}
          tag: ${{ github.ref }}

      - name: Upload src and pgo data to release
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: ${{ env.ASSET }}
          asset_name: ${{ env.ASSET }}
          tag: ${{ github.ref }}

      - name: Upload binaries to release
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: target/release/${{ matrix.artifact_name }}
          asset_name: ${{ matrix.asset_name }}
          tag: ${{ github.ref }}


text:
use anyhow::{Context, Error, Result};
use clap::Parser;
use env_logger::Env;
use flate2::Compression;
use git_version::git_version;
use grep_cli::{stdout, unescape};
use gzp::{deflate::Bgzf, ZBuilder};
use hcklib::{
    core::{Core, CoreConfig, CoreConfigBuilder, HckInput},
    field_range::RegexOrString,
    line_parser::{RegexLineParser, SubStrLineParser},
    mmap::MmapChoice,
};
use lazy_static::lazy_static;
use log::{error, warn};
use regex::bytes::Regex;
use ripline::{
    line_buffer::{LineBuffer, LineBufferBuilder},
    LineTerminator,
};
use std::{
    fs::File,
    io::{self, BufWriter, Write},
    path::{Path, PathBuf},
    process::exit,
};
use termcolor::ColorChoice;

lazy_static! {
    /// Default number of compression threads to use.
    ///
    /// This will be 4 if >= 4 threads are present, otherwise it will
    /// be num_cpus - 1.
    pub static ref DEFAULT_CPUS: String = {
        let num_cores = num_cpus::get();
        if num_cores < 4 {
            num_cores.saturating_sub(1)
        } else {
            4
        }.to_string()
    };
}

pub const HCK_VERSION: &str = git_version!(
    cargo_prefix = "cargo:",
    prefix = "git:",
    // Note that on the CLI, the v* needs to be in single quotes
    // When passed here though there seems to be some magic quoting that happens.
    args = ["--always", "--dirty=-modified", "--match=v*"]
);

/// Determine if we should write to a file or stdout.
fn select_output<P: AsRef<Path>>(output: Option<P>) -> Result<Box<dyn Write + Send + 'static>> {
    let writer: Box<dyn Write + Send + 'static> = match output {
        Some(path) => {
            if path.as_ref().as_os_str() == "-" {
                // TODO: verify that stdout buffers when writing to a terminal now (this was a bug in Rust at some point).
                Box::new(stdout(ColorChoice::Never))
            } else {
                Box::new(File::create(&path).with_context(|| {
                    format!("Failed to open {} for writing.", path.as_ref().display())
                })?)
            }
        }
        None => Box::new(stdout(ColorChoice::Never)),
    };
    Ok(writer)
}

/// Check if err is a broken pipe.
#[inline]
fn is_broken_pipe(err: &Error) -> bool {
    if let Some(io_err) = err.downcast_ref::<io::Error>() {
        if io_err.kind() == io::ErrorKind::BrokenPipe {
            return true;
        }
    }
    false
}
/// * `delimiter` is a regex by default and a fixed substring with `-L`
/// * `header-fields` allows for specifying a literal or a regex to match header names to select columns
/// * both `header-fields` and `fields` order dictate the order of the output columns
/// * input files (not stdin) are automatically compressed
/// * the output delimiter can specified with `-D`
///
/// ## Selection by headers
///
/// Instead of specifying fields to output by index ranges (i.e `1-2,4-`), you can specify a regex or string literal
/// to select a headered column to output with the `-F` option. By default `-F` options are treated as string literals.
/// To treat them as regexs add the `-r` flag.
///
/// ## Ordering of outputs
///
/// *Values are written only once*. So for a `fields` value of `4-,1,5-8`, which translates to "print columns 4 through
/// the end and then the last column and then columns 5 through 8", columns 5-8 won't be printed again because they
/// were already consumed by the `4-` range.
///
/// If `field-headers` is used as a regex then the headers will be be grouped together in groups that all matched the
/// same regex, and in the order of the regex as specified on the CLI.
#[derive(Debug, Parser)]
#[clap(author, version = HCK_VERSION)]
struct Opts {
    /// Input files to parse, defaults to stdin.
    ///
    /// If a file has a recognizable file extension indicating that it is compressed, and a local binary
    /// to perform decompression is found, decompression will occur automagically. This requires with `-z`.
    input: Vec<PathBuf>,

    /// Output file to write to, defaults to stdout
    #[clap(short, long, allow_hyphen_values = true)]
    output: Option<PathBuf>,

    /// Delimiter to use on input files, this is a substring literal by default. To treat it as a literal add the `-L` flag.
    #[clap(short, long, default_value = r"\s+", allow_hyphen_values = true)]
    delimiter: String,

    /// Treat the delimiter as a string literal. This can significantly improve performance, especially for single byte delimiters.
    #[clap(short = 'L', long)]
    delim_is_literal: bool,

    /// Use the input delimiter as the output delimiter if the input is literal and no other output delimiter has been set.
    #[clap(
        short = 'I',
        long,
        requires("delim-is-literal"),
        conflicts_with("output-delimiter")
    )]
    use_input_delim: bool,

    /// Delimiter string to use on outputs
    #[clap(short = 'D', long, default_value = "\t", allow_hyphen_values = true)]
    output_delimiter: String,

    /// Fields to keep in the output, ex: 1,2-,-5,2-5. Fields are 1-based and inclusive.
    #[clap(short, long, allow_hyphen_values = true)]
    fields: Option<String>,

    /// Fields to exclude from the output, ex: 3,9-11,15-. Exclude fields are 1 based and inclusive.
    /// Exclude fields take precedence over `fields`.
    #[clap(short = 'e', long, allow_hyphen_values = true)]
    exclude: Option<String>,

    /// Headers to exclude from the output, ex: '^badfield.*$`. This is a string literal by default.
    /// Add the `-r` flag to treat as a regex.
    #[clap(short = 'E', long, number_of_values = 1, allow_hyphen_values = true)]
    exclude_header: Option<Vec<Regex>>,

    /// A string literal or regex to select headers, ex: '^is_.*$`. This is a string literal
    /// by default. add the `-r` flag to treat it as a regex.
    #[clap(short = 'F', long, number_of_values = 1, allow_hyphen_values = true)]
    header_field: Option<Vec<Regex>>,

    /// Treat the header_fields as regexs instead of string literals
    #[clap(short = 'r', long)]
    header_is_regex: bool,

    /// Try to find the correct decompression method based on the file extensions
    #[clap(short = 'z', long)]
    try_decompress: bool,

    /// Try to gzip compress the output
    #[clap(short = 'Z', long)]
    try_compress: bool,

    /// Threads to use for compression, 0 will result in `hck` staying single threaded.
    #[clap(short = 't', long, default_value=&DEFAULT_CPUS.as_str())]
    compression_threads: usize,

    /// Compression level
    #[clap(short = 'l', long, default_value = "6")]
    compression_level: u32,

    /// Disallow the possibility of using mmap
    #[clap(long)]
    no_mmap: bool,

    /// Support CRLF newlines
    #[clap(long)]
    crlf: bool,
}

fn main() -> Result<()> {
    // TODO: move tests / add more tests
    env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
    let opts = Opts::parse();

    let writer = select_output(opts.output.as_ref())?;
    // TODO: Support all flate2 compression targets via enum on `-Z`
    let mut writer: Box<dyn Write> = if opts.try_compress {
        Box::new(
            ZBuilder::<Bgzf, _>::new()
                .compression_level(Compression::new(opts.compression_level))
                .num_threads(opts.compression_threads)
                .from_writer(writer),
        )
    } else {
        Box::new(BufWriter::new(writer))
    };

    if opts.input.is_empty() && opts.try_decompress && opts.header_field.is_some() {
        warn!("Selections based on header fields is not currently supported on STDIN compressed data.");
    }

    let inputs: Vec<HckInput<PathBuf>> = if opts.input.is_empty() {
        vec![HckInput::Stdin]
    } else {
        opts.input
            .iter()
            .map(|p| {
                if p.as_os_str() == "-" {
                    HckInput::Stdin
                } else {
                    HckInput::Path(p.clone())
                }
            })
            .collect()
    };

    let mut conf_builder = CoreConfigBuilder::new();

    let line_term = if opts.crlf {
        LineTerminator::crlf()
    } else {
        LineTerminator::default()
    };
    conf_builder = conf_builder.line_terminator(line_term);

    let mmap = if opts.no_mmap {
        MmapChoice::never()
    } else {
        unsafe { MmapChoice::auto() }
    };

    let out_delim = if opts.delim_is_literal && opts.use_input_delim {
        unescape(&opts.delimiter)
    } else {
        unescape(&opts.output_delimiter)
    };

    let conf = conf_builder
        .mmap(mmap)
        .delimiter(opts.delimiter.as_bytes())
        .output_delimiter(&out_delim)
        .is_regex_parser(!opts.delim_is_literal)
        .try_decompress(opts.try_decompress)
        .fields(opts.fields.as_deref())
        .headers(opts.header_field.as_deref())
        .exclude(opts.exclude.as_deref())
        .exclude_headers(opts.exclude_header.as_deref())
        .header_is_regex(opts.header_is_regex)
        .build()?;

    let mut line_buffer = LineBufferBuilder::new().build();

    for input in inputs.into_iter() {
        if let Err(err) = run(input, &mut writer, &conf, &mut line_buffer) {
            if is_broken_pipe(&err) {
                exit(0)
            }
            error!("{}", err);
            exit(1)
        }
    }
    Ok(())
}

/// Run the actual parsing and writing
fn run<W: Write>(
    input: HckInput<PathBuf>,
    writer: &mut W,
    conf: &CoreConfig,
    line_buffer: &mut LineBuffer,
) -> Result<()> {
    let (extra, fields) = conf.parse_fields(&input)?;
    // No point processing empty fields
    if fields.is_empty() {
        return Ok(());
    }

    match conf.parsed_delim() {
        RegexOrString::Regex(regex) => {
            let mut core = Core::new(
                conf,
                &fields,
                RegexLineParser::new(&fields, regex),
                line_buffer,
            );
            core.hck_input(input, writer, extra)?;
        }
        RegexOrString::String(s) => {
            // let s = unescape(s);
            let mut core = Core::new(
                conf,
                &fields,
                SubStrLineParser::new(&fields, s.as_bytes()),
                line_buffer,
            );
            core.hck_input(input, writer, extra)?;
        }
    };
    Ok(())
}

#[cfg(test)]
mod test {

    use std::io::BufReader;

    use super::*;
    use bstr::io::BufReadExt;
    use rstest::rstest;
    use tempfile::TempDir;

    /// Build a set of opts for testing
    fn build_opts(
        input_file: impl AsRef<Path>,
        output_file: impl AsRef<Path>,
        fields: &str,
        no_mmap: bool,
        delimiter: &str,
    ) -> Opts {
        Opts {
            input: vec![input_file.as_ref().to_path_buf()],
            output: Some(output_file.as_ref().to_path_buf()),
            delimiter: delimiter.to_string(),
            delim_is_literal: false,
            output_delimiter: "\t".to_owned(),
            use_input_delim: false,
            fields: Some(fields.to_owned()),
            header_field: None,
            header_is_regex: true,
            try_decompress: false,
            try_compress: false,
            no_mmap,
            crlf: false,
            exclude: None,
            exclude_header: None,
            compression_level: 3,
            compression_threads: 0,
        }
    }

    /// Build a set of opts for testing
    fn build_opts_not_regex(
        input_file: impl AsRef<Path>,
        output_file: impl AsRef<Path>,
        fields: &str,
        no_mmap: bool,
        delimiter: &str,
    ) -> Opts {
        Opts {
            input: vec![input_file.as_ref().to_path_buf()],
            output: Some(output_file.as_ref().to_path_buf()),
            delimiter: delimiter.to_string(),
            delim_is_literal: true,
            output_delimiter: "\t".to_owned(),
            use_input_delim: false,
            fields: Some(fields.to_owned()),
            header_field: None,
            header_is_regex: true,
            try_decompress: false,
            try_compress: false,
            no_mmap,
            crlf: false,
            exclude: None,
            exclude_header: None,
            compression_level: 3,
            compression_threads: 0,
        }
    }

    /// Build a set of opts for testing
    #[allow(clippy::too_many_arguments)]
    fn build_opts_generic(
        input_file: impl AsRef<Path>,
        output_file: impl AsRef<Path>,
        fields: Option<&str>,
        header_field: Option<Vec<Regex>>,
        exclude: Option<&str>,
        no_mmap: bool,
        delimiter: &str,
        delim_is_literal: bool,
        header_is_regex: bool,
    ) -> Opts {
        Opts {
            input: vec![input_file.as_ref().to_path_buf()],
            output: Some(output_file.as_ref().to_path_buf()),
            delimiter: delimiter.to_string(),
            delim_is_literal,
            output_delimiter: "\t".to_owned(),
            use_input_delim: false,
            fields: fields.map(|f| f.to_owned()),
            header_field,
            header_is_regex,
            try_decompress: false,
            try_compress: false,
            no_mmap,
            crlf: false,
            exclude: exclude.map(|e| e.to_owned()),
            exclude_header: None,
            compression_threads: 0,
            compression_level: 3,
        }
    }

    /// Simple function to read a tsv into a nested list of lists.
    fn read_tsv(path: impl AsRef<Path>) -> Vec<Vec<String>> {
        let reader = BufReader::new(File::open(path).unwrap());
        let mut result = vec![];
        let r = Regex::new(r"\s+").unwrap();

        for line in reader.byte_lines() {
            let line = &line.unwrap();
            result.push(
                r.split(line)
                    .map(|s| unsafe { String::from_utf8_unchecked(s.to_vec()) })
                    .collect(),
            );
        }
        result
    }

    /// Write delmited data to a file.
    fn write_file(path: impl AsRef<Path>, data: Vec<Vec<&str>>, sep: &str) {
        let mut writer = BufWriter::new(File::create(path).unwrap());
        for row in data {
            writeln!(&mut writer, "{}", row.join(sep)).unwrap();
        }
        writer.flush().unwrap();
    }

    // Wrap the run function to create the readers and writers.
    fn run_wrapper<P: AsRef<Path>>(input: P, output: P, opts: &Opts) {
        let conf = CoreConfigBuilder::new()
            .delimiter(opts.delimiter.as_bytes())
            .is_regex_parser(!opts.delim_is_literal)
            .mmap(if opts.no_mmap {
                MmapChoice::never()
            } else {
                unsafe { MmapChoice::auto() }
            })
            .output_delimiter(opts.output_delimiter.as_bytes())
            .headers(opts.header_field.as_deref())
            .fields(opts.fields.as_deref())
            .exclude(opts.exclude.as_deref())
            .exclude_headers(opts.exclude_header.as_deref())
            .header_is_regex(opts.header_is_regex)
            .build()
            .unwrap();
        let mut line_buffer = LineBufferBuilder::new().build();
        let mut writer = BufWriter::new(File::create(output).unwrap());
        run(
            HckInput::Path(input.as_ref().to_owned()),
            &mut writer,
            &conf,
            &mut line_buffer,
        )
        .unwrap();
    }

    const FOURSPACE: &str = "    ";

    #[rstest]
    fn test_exclude_one(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("1,3"),
            None,
            Some("3"),
            no_mmap,
            hck_delim,
            delim_is_literal,
            false,
        );
        let data = vec![vec!["a", "b", "c"], vec!["1", "2", "3"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["a",], vec!["1"]]);
    }

    #[rstest]
    fn test_exclude_range_overlap_front(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("3-"),
            None,
            Some("-5"),
            no_mmap,
            hck_delim,
            delim_is_literal,
            false,
        );
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f"],
            vec!["1", "2", "3", "4", "5", "6"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["f",], vec!["6"]]);
    }

    #[rstest]
    fn test_exclude_range_overlap_back(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("2-5"),
            None,
            Some("3-"),
            no_mmap,
            hck_delim,
            delim_is_literal,
            false,
        );
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f"],
            vec!["1", "2", "3", "4", "5", "6"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["b",], vec!["2"]]);
    }

    #[rstest]
    fn test_exclude_range_split_fields(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("1-"),
            None,
            Some("3-5"),
            no_mmap,
            hck_delim,
            delim_is_literal,
            false,
        );
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f"],
            vec!["1", "2", "3", "4", "5", "6"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["a", "b", "f"], vec!["1", "2", "6"]]);
    }

    #[rstest]
    fn test_exclude_range_all(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("4,3"),
            None,
            Some("2-5"),
            no_mmap,
            hck_delim,
            delim_is_literal,
            false,
        );
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f"],
            vec!["1", "2", "3", "4", "5", "6"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert!(filtered.is_empty());
    }

    #[rstest]
    fn test_exclude_range_split_fields_reorder(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("4-6,1-3"),
            None,
            Some("3-5"),
            no_mmap,
            hck_delim,
            delim_is_literal,
            false,
        );
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f"],
            vec!["1", "2", "3", "4", "5", "6"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["f", "a", "b"], vec!["6", "1", "2"]]);
    }

    #[rstest]
    fn test_headers_simple(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
        #[values(true, false)] header_is_regex: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            None,
            Some(vec![Regex::new("a").unwrap()]),
            None,
            no_mmap,
            hck_delim,
            delim_is_literal,
            header_is_regex,
        );
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f"],
            vec!["1", "2", "3", "4", "5", "6"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["a"], vec!["1"]]);
    }

    #[rstest]
    fn test_headers_simple2(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
        #[values(true, false)] header_is_regex: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            None,
            Some(vec![Regex::new("a").unwrap(), Regex::new("c").unwrap()]),
            None,
            no_mmap,
            hck_delim,
            delim_is_literal,
            header_is_regex,
        );
        let data = vec![vec!["a", "b", "c"], vec!["1", "2", "3"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["a", "c"], vec!["1", "3"]]);
    }

    #[rstest]
    fn test_duplicate_field_selection_more(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
        #[values(true, false)] header_is_regex: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("3,3,1,2"),
            None,
            None,
            no_mmap,
            hck_delim,
            delim_is_literal,
            header_is_regex,
        );
        let data = vec![vec!["a", "b", "c", "d", "e"], vec!["1", "2", "3", "4", "5"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["c", "a", "b"], vec!["3", "1", "2"]]);
    }

    #[rstest]
    fn test_duplicate_field_selection_range(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
        #[values(true, false)] header_is_regex: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("2-3,5,1,2-4"),
            None,
            None,
            no_mmap,
            hck_delim,
            delim_is_literal,
            header_is_regex,
        );
        let data = vec![vec!["a", "b", "c", "d", "e"], vec!["1", "2", "3", "4", "5"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(
            filtered,
            vec![vec!["b", "c", "e", "a", "d"], vec!["2", "3", "5", "1", "4"]]
        );
    }

    #[rstest]
    fn test_headers_and_fields(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
        #[values(true, false)] header_is_regex: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("3"),
            Some(vec![Regex::new("b").unwrap(), Regex::new("a").unwrap()]),
            None,
            no_mmap,
            hck_delim,
            delim_is_literal,
            header_is_regex,
        );
        let data = vec![vec!["a", "b", "c", "d", "e"], vec!["1", "2", "3", "4", "5"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["b", "c", "a"], vec!["2", "3", "1"]]);
    }

    #[rstest]
    fn test_duplicate_field_selection(
        #[values(true, false)] no_mmap: bool,
        #[values(r" ", "  ")] hck_delim: &str,
        #[values(true, false)] delim_is_literal: bool,
        #[values(true, false)] header_is_regex: bool,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_generic(
            &input_file,
            &output_file,
            Some("3,1,3"),
            None,
            None,
            no_mmap,
            hck_delim,
            delim_is_literal,
            header_is_regex,
        );
        let data = vec![vec!["a", "b", "c", "d"], vec!["1", "2", "3", "4"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["c", "a"], vec!["3", "1"]]);
    }
    #[rstest]
    #[rustfmt::skip::macros(vec)]
    fn test_read_single_values(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "1", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c"],
            vec!["1", "2", "3"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["a"], vec!["1"]]);
    }

    #[rstest]
    fn test_read_several_single_values(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "1,3", no_mmap, hck_delim);
        let data = vec![vec!["a", "b", "c"], vec!["1", "2", "3"]];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["a", "c"], vec!["1", "3"]]);
    }

    #[rstest]
    fn test_read_several_single_values_with_invalid_utf8(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "1,3", no_mmap, hck_delim);
        let bad_str = unsafe { String::from_utf8_unchecked(b"a\xED\xA0\x80z".to_vec()) };
        let data = vec![vec![bad_str.as_str(), "b", "c"], vec!["1", "2", "3"]];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec![bad_str.as_str(), "c"], vec!["1", "3"]]);
    }

    #[rstest]
    fn test_read_single_range(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "2-", no_mmap, hck_delim);
        let data = vec![vec!["a", "b", "c", "d"], vec!["1", "2", "3", "4"]];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["b", "c", "d"], vec!["2", "3", "4"]]);
    }

    #[rstest]
    fn test_read_serveral_range(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "2-4,6-", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(
            filtered,
            vec![vec!["b", "c", "d", "f", "g"], vec!["2", "3", "4", "6", "7"]]
        );
    }

    #[rstest]
    fn test_read_mixed_fields1(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "2,4-", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(
            filtered,
            vec![vec!["b", "d", "e", "f", "g"], vec!["2", "4", "5", "6", "7"]]
        );
    }

    #[rstest]
    fn test_read_mixed_fields2(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "-4,7", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(
            filtered,
            vec![vec!["a", "b", "c", "d", "g"], vec!["1", "2", "3", "4", "7"]]
        );
    }

    #[rstest]
    fn test_read_no_delimis_found(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "-4,7", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, "-");
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // We hae no concept of only-delimited, so if no delim is found the whole line
        // is treated as column 1.
        assert_eq!(filtered, vec![vec!["a-b-c-d-e-f-g"], vec!["1-2-3-4-5-6-7"]]);
    }

    #[rstest]
    fn test_read_over_end(#[values(true, false)] no_mmap: bool, #[values(r"\s+")] hck_delim: &str) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "-4,8,11-", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![vec!["a", "b", "c", "d"], vec!["1", "2", "3", "4"]]
        );
    }

    #[rstest]
    fn test_reorder1(#[values(true, false)] no_mmap: bool, #[values(r"\s+")] hck_delim: &str) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "6,-4", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![vec!["f", "a", "b", "c", "d"], vec!["6", "1", "2", "3", "4"]]
        );
    }

    #[rstest]
    fn test_reorder_merged_range(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts(&input_file, &output_file, "1,3,2,7,6", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![vec!["a", "c", "b", "g", "f"], vec!["1", "3", "2", "7", "6"]]
        );
    }

    #[rstest]
    fn test_reorder2(#[values(true, false)] no_mmap: bool, #[values(r"\s+")] hck_delim: &str) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        // 4-5 should not be repeated at the end and only written once.
        let opts = build_opts(&input_file, &output_file, "3-,1,4-5", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, FOURSPACE);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![
                vec!["c", "d", "e", "f", "g", "a"],
                vec!["3", "4", "5", "6", "7", "1"]
            ]
        );
    }

    #[rstest]
    #[rustfmt::skip::macros(vec)]
    fn test_read_single_values_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "1", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c"],
            vec!["1", "2", "3"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["a"], vec!["1"]]);
    }

    #[rstest]
    fn test_read_several_single_values_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "1,3", no_mmap, hck_delim);
        let data = vec![vec!["a", "b", "c"], vec!["1", "2", "3"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["a", "c"], vec!["1", "3"]]);
    }

    #[rstest]
    fn test_read_several_single_values_with_invalid_utf8_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "1,3", no_mmap, hck_delim);
        let bad_str = unsafe { String::from_utf8_unchecked(b"a\xED\xA0\x80z".to_vec()) };
        let data = vec![vec![bad_str.as_str(), "b", "c"], vec!["1", "2", "3"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec![bad_str.as_str(), "c"], vec!["1", "3"]]);
    }

    #[rstest]
    fn test_read_single_range_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "2-", no_mmap, hck_delim);
        let data = vec![vec!["a", "b", "c", "d"], vec!["1", "2", "3", "4"]];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(filtered, vec![vec!["b", "c", "d"], vec!["2", "3", "4"]]);
    }

    #[rstest]
    fn test_read_serveral_range_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "2-4,6-", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(
            filtered,
            vec![vec!["b", "c", "d", "f", "g"], vec!["2", "3", "4", "6", "7"]]
        );
    }

    #[rstest]
    fn test_read_mixed_fields1_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "2,4-", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(
            filtered,
            vec![vec!["b", "d", "e", "f", "g"], vec!["2", "4", "5", "6", "7"]]
        );
    }

    #[rstest]
    fn test_read_mixed_fields2_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "-4,7", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        assert_eq!(
            filtered,
            vec![vec!["a", "b", "c", "d", "g"], vec!["1", "2", "3", "4", "7"]]
        );
    }

    #[rstest]
    fn test_read_no_delimis_found_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "-4,7", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, "-");
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // We hae no concept of only-delimited, so if no delim is found the whole line
        // is treated as column 1.
        assert_eq!(filtered, vec![vec!["a-b-c-d-e-f-g"], vec!["1-2-3-4-5-6-7"]]);
    }

    #[rstest]
    fn test_read_over_end_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "-4,8,11-", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![vec!["a", "b", "c", "d"], vec!["1", "2", "3", "4"]]
        );
    }

    #[rstest]
    fn test_reorder1_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        let opts = build_opts_not_regex(&input_file, &output_file, "6,-4", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![vec!["f", "a", "b", "c", "d"], vec!["6", "1", "2", "3", "4"]]
        );
    }

    #[rstest]
    fn test_reorder2_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        // 4-5 should not be repeated at the end and only written once.
        let opts = build_opts_not_regex(&input_file, &output_file, "3-,1,4-5", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![
                vec!["c", "d", "e", "f", "g", "a"],
                vec!["3", "4", "5", "6", "7", "1"]
            ]
        );
    }

    /// Tests from users
    #[rstest]
    fn test_reorder_no_split_found(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        // 4-5 should not be repeated at the end and only written once.
        let opts = build_opts_not_regex(&input_file, &output_file, "3-,1,4-5", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, "-");
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(filtered, vec![vec!["a-b-c-d-e-f-g"], vec!["1-2-3-4-5-6-7"]]);
    }

    /// Tests from users
    #[rstest]
    fn test_reorder_no_split_found_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        // 4-5 should not be repeated at the end and only written once.
        let opts = build_opts(&input_file, &output_file, "3-,1,4-5", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, "---");
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![
                vec!["a---b---c---d---e---f---g"],
                vec!["1---2---3---4---5---6---7"]
            ]
        );
    }

    #[rstest]
    fn test_issue_12_with_regex(
        #[values(true, false)] no_mmap: bool,
        #[values(r"\s+")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        // 4-5 should not be repeated at the end and only written once.
        let opts = build_opts(&input_file, &output_file, "2,3,4-", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, "  ");
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![
                vec!["b", "c", "d", "e", "f", "g"],
                vec!["2", "3", "4", "5", "6", "7"]
            ]
        );
    }

    #[rstest]
    fn test_issue_12_no_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        // 4-5 should not be repeated at the end and only written once.
        let opts = build_opts(&input_file, &output_file, "2,3,4-", no_mmap, hck_delim);
        let data = vec![
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![
                vec!["b", "c", "d", "e", "f", "g"],
                vec!["2", "3", "4", "5", "6", "7"]
            ]
        );
    }

    #[rstest]
    fn test_issue_38_not_regex(
        #[values(true, false)] no_mmap: bool,
        #[values("    ", " ")] hck_delim: &str,
    ) {
        let tmp = TempDir::new().unwrap();
        let input_file = tmp.path().join("input.txt");
        let output_file = tmp.path().join("output.txt");
        // 4-5 should not be repeated at the end and only written once.
        let opts = build_opts_not_regex(&input_file, &output_file, "1,2", no_mmap, hck_delim);
        let data = vec![
            vec![""],
            vec![""],
            vec!["a", "b", "c", "d", "e", "f", "g"],
            vec![""],
            vec![""],
            vec!["1", "2", "3", "4", "5", "6", "7"],
        ];
        write_file(&input_file, data, hck_delim);
        run_wrapper(&input_file, &output_file, &opts);
        let filtered = read_tsv(output_file);

        // columns past end in fields are ignored
        assert_eq!(
            filtered,
            vec![
                vec![""],
                vec![""],
                vec!["a", "b"],
                vec![""],
                vec![""],
                vec!["1", "2"]
            ]
        );
    }
}