bookdata/io/
mod.rs

1use friendly::bytes;
2use indicatif::ProgressBar;
3use log::*;
4use std::fs;
5use std::io::{BufRead, BufReader, Result as IOResult};
6use std::path::{Path, PathBuf};
7
8pub mod background;
9pub mod compress;
10pub mod ext;
11pub mod lines;
12pub mod object;
13
14pub use compress::open_gzin_progress;
15pub use lines::LineProcessor;
16pub use object::ObjectWriter;
17
18/// Trait for data processing sinks with input and ouptut files.
19pub trait DataSink {
20    /// Get the output files for the sink.
21    fn output_files(&self) -> Vec<PathBuf>;
22
23    /// Get auxillary input files for the sink.
24    ///
25    /// Most sinks are also an [ObjectWriter], and the primary input is written
26    /// to the sink; that input file is not reported here.  However, sinks may
27    /// require additional input files to process, and those files can be reported
28    /// here.
29    fn input_files(&self) -> Vec<PathBuf> {
30        Vec::new()
31    }
32}
33
34/// Log the sizes of a set of files.
35pub fn log_file_info<P: AsRef<Path>, S: IntoIterator<Item = P>>(files: S) -> IOResult<()> {
36    for path in files {
37        let path = path.as_ref();
38        let size = file_size(path)?;
39        info!("output {:?}: {}", path, bytes(size));
40    }
41
42    Ok(())
43}
44
45/// Convert a list of strings into owned [PathBuf]s.
46pub fn path_list(paths: &[&str]) -> Vec<PathBuf> {
47    paths.into_iter().map(|p| PathBuf::from(p)).collect()
48}
49
50/// Get the size of a file.
51pub fn file_size<P: AsRef<Path>>(path: P) -> IOResult<u64> {
52    let meta = fs::metadata(path)?;
53    Ok(meta.len())
54}
55
56/// Open a file as a buffered reader with a progress bar.
57pub fn open_progress(path: &Path, pb: ProgressBar) -> IOResult<impl BufRead> {
58    let name = path.file_name().unwrap().to_string_lossy();
59    let read = fs::File::open(path)?;
60    pb.set_length(read.metadata()?.len());
61    pb.set_prefix(name.to_string());
62
63    let read = pb.wrap_read(read);
64    let read = BufReader::new(read);
65    Ok(read)
66}
bookdata/io/mod.rs

bookdata/io/
mod.rs
