fogti/psdn-tacsvs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

use rayon

Browse source
This commit is contained in:
Erik Zscheile 2019-12-23 21:17:21 +01:00
parent 6b11a8a2a5
commit fc77a1cc4b
3 changed files with 194 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

129
Cargo.lock generated
View file

@ -133,6 +133,39 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "995a44c877f9212528ccc74b21a232f66ad69001e40ede5bcee2ac9ef2657120"
[[package]]
name = "crossbeam-deque"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils 0.7.0",
]
[[package]]
name = "crossbeam-epoch"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils 0.7.0",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfd6515864a82d2f877b42813d4553292c6659498c9a2aa31bab5a15243c2700"
dependencies = [
"crossbeam-utils 0.7.0",
]
[[package]]
name = "crossbeam-utils"
version = "0.6.6"
@ -143,6 +176,17 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
dependencies = [
"autocfg",
"cfg-if",
"lazy_static",
]
[[package]]
name = "csv"
version = "1.1.1"
@ -301,6 +345,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "hermit-abi"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f629dc602392d3ec14bfc8a09b5e644d7ffd725102b48b81e59f90f2633621d7"
dependencies = [
"libc",
]
[[package]]
name = "itertools"
version = "0.8.2"
@ -347,6 +400,15 @@ dependencies = [
"winapi",
]
[[package]]
name = "memoffset"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
dependencies = [
"rustc_version",
]
[[package]]
name = "num-integer"
version = "0.1.41"
@ -366,6 +428,16 @@ dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "prettytable-rs"
version = "0.8.0"
@ -408,6 +480,7 @@ dependencies = [
"fixed",
"itertools",
"prettytable-rs",
"rayon",
"readfilez",
"ron",
"serde",
@ -462,6 +535,30 @@ dependencies = [
"winapi",
]
[[package]]
name = "rayon"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098"
dependencies = [
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9"
dependencies = [
"crossbeam-deque",
"crossbeam-queue",
"crossbeam-utils 0.7.0",
"lazy_static",
"num_cpus",
]
[[package]]
name = "rdrand"
version = "0.4.0"
@ -527,7 +624,7 @@ checksum = "4ca4eaef519b494d1f2848fc602d18816fed808a981aedf4f1f00ceb7c9d32cf"
dependencies = [
"base64",
"blake2b_simd",
"crossbeam-utils",
"crossbeam-utils 0.6.6",
]
[[package]]
@ -536,12 +633,42 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
"semver",
]
[[package]]
name = "ryu"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
[[package]]
name = "scopeguard"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
[[package]]
name = "semver"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
"semver-parser",
]
[[package]]
name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
version = "1.0.104"

3
Cargo.toml
View file

@ -9,13 +9,14 @@ chrono = { version = "0.4", features = ["serde"] }
csv = "1.1"
encoding = "0.2"
fixed = { version = "0.5", features = ["serde", "std"] }
itertools = "0.8"
prettytable = { version = "0.8", package = "prettytable-rs" }
rayon = "1.3"
readfilez = "0.2"
ron = "0.5"
serde = { version = "1.0", features = ["derive"] }
term = "0.5"
thiserror = "1.0"
itertools = "0.8.2"
[profile.release]
lto = true

115
src/main.rs
View file

@ -10,14 +10,17 @@ fn main() {
return;
}
use chrono::Datelike;
use encoding::types::Encoding;
use prettytable::{cell, format, row, Table};
use std::{
collections::{BTreeMap, HashSet},
io::Write,
use {
chrono::Datelike,
encoding::types::Encoding,
itertools::Itertools,
prettytable::{cell, format, row, Table},
rayon::prelude::*,
std::{
collections::{BTreeMap, HashSet},
io::Write,
},
};
use itertools::Itertools;
let mut highlight = HashSet::new();
@ -30,57 +33,61 @@ fn main() {
highlight.remove(";");
}
let mut dat: Vec<_> = args.into_iter().flat_map(|i| {
let fh = readfilez::read_from_file(std::fs::File::open(i)).expect("unable to open file");
let mut dat: Vec<_> = args
.into_par_iter()
.flat_map(|i| {
let fh =
readfilez::read_from_file(std::fs::File::open(i)).expect("unable to open file");
let tmp = encoding::all::ISO_8859_1
.decode(&*fh, encoding::types::DecoderTrap::Replace)
.expect("got invalid latin-1 data");
let tmp = encoding::all::ISO_8859_1
.decode(&*fh, encoding::types::DecoderTrap::Replace)
.expect("got invalid latin-1 data");
std::mem::drop(fh);
std::mem::drop(fh);
let mut rdr = csv::ReaderBuilder::new()
.delimiter(b';')
.flexible(true)
.has_headers(false)
.from_reader(tmp.as_bytes());
let mut rdr = csv::ReaderBuilder::new()
.delimiter(b';')
.flexible(true)
.has_headers(false)
.from_reader(tmp.as_bytes());
let mut recsit = rdr.records().skip(8);
let mut recsit = rdr.records().skip(8);
assert_eq!(
recsit.next().unwrap().unwrap(),
vec![
"Buchungstag",
"Valuta",
"Auftraggeber/Zahlungsempfänger",
"Empfänger/Zahlungspflichtiger",
"Konto-Nr.",
"IBAN",
"BLZ",
"BIC",
"Vorgang/Verwendungszweck",
"Kundenreferenz",
"Währung",
"Umsatz",
" "
]
);
assert_eq!(
recsit.next().unwrap().unwrap(),
vec![
"Buchungstag",
"Valuta",
"Auftraggeber/Zahlungsempfänger",
"Empfänger/Zahlungspflichtiger",
"Konto-Nr.",
"IBAN",
"BLZ",
"BIC",
"Vorgang/Verwendungszweck",
"Kundenreferenz",
"Währung",
"Umsatz",
" "
]
);
let mut idat = Vec::new();
for result in recsit {
let record = result.expect("got invalid line");
let record_bak = record.clone();
let pres: Result<transaction::TransactionLine, _> =
std::convert::TryInto::try_into(record);
let mut idat = Vec::new();
for result in recsit {
let record = result.expect("got invalid line");
let record_bak = record.clone();
let pres: Result<transaction::TransactionLine, _> =
std::convert::TryInto::try_into(record);
match pres {
Ok(tl) => idat.push(tl),
Err(transaction::ParseError::Finalizer) => break,
Err(x) => panic!("got error '{}' @ {:?}", x, record_bak),
match pres {
Ok(tl) => idat.push(tl),
Err(transaction::ParseError::Finalizer) => break,
Err(x) => panic!("got error '{}' @ {:?}", x, record_bak),
}
}
}
idat
}).collect();
idat
})
.collect();
let mut stdout = term::stdout().unwrap();
@ -115,7 +122,13 @@ fn main() {
ent.1 += i.umsatz;
}
writeln!(&mut stdout, "skipped {} duplicates and {} misc entries", oldlen - newlen, i_skipped).unwrap();
writeln!(
&mut stdout,
"skipped {} duplicates and {} misc entries",
oldlen - newlen,
i_skipped
)
.unwrap();
if accu.is_empty() {
return;