diff --git a/Cargo.toml b/Cargo.toml index b0d56c9..2d1a2d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,12 +17,13 @@ polars = { version = "0.30", features = [ reqwest = { version = "0.11", features = ["blocking"] } glob = { version = "0.3" } clokwerk = "0.4.0" -strum_macros = "0.24" -strum = "0.24" +strum_macros = "0.25" +strum = "0.25" tokio = { version = "1.26", features = ["full"] } openssl = { version = "0.10", features = ["vendored"] } chrono = { version = "0.4", features = ["serde"] } serde_json = "1.0" +rand = "0.8" [dev-dependencies] serial_test = "*" diff --git a/Dockerfile b/Dockerfile index 6e62716..3e55d26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,29 @@ -FROM rust:latest AS builder +FROM rust:bookworm AS builder -RUN rustup target add x86_64-unknown-linux-musl && apt update && apt install -y musl-tools musl-dev +RUN apt-get update && \ + apt install -y musl-tools musl-dev libssl-dev clang mold + +# RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin +RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C /usr/local/bin +# RUN cargo install cargo-nextest --locked WORKDIR /ark-invest-api-rust-data COPY . . +RUN rustup target add x86_64-unknown-linux-musl && rustup update && cargo update + RUN --mount=type=cache,target=/usr/local/cargo,from=rust,source=/usr/local/cargo \ --mount=type=cache,target=./target \ cargo build --target x86_64-unknown-linux-musl --release && \ - cp ./target/x86_64-unknown-linux-musl/release/ark-invest-api-rust-data . + cp ./target/target/x86_64-unknown-linux-musl/release/ark-invest-api-rust-data . + +FROM builder AS test + +RUN --mount=type=cache,target=/usr/local/cargo,from=rust,source=/usr/local/cargo \ + --mount=type=cache,target=./target \ + cargo nextest run --release --target x86_64-unknown-linux-musl \ + -E "all() - test(get_api) - kind(bin)" FROM alpine:latest diff --git a/docker-compose.yml b/docker-compose.yml index 3cc4a4c..c280888 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,15 +3,15 @@ services: ark-invest-api-rust-data: # image: ghcr.io/NexVeridian/ark-invest-api-rust-data:latest image: ark-invest-api-rust-data + container_name: ark-invest-api-rust-data build: context: . + target: test args: DOCKER_BUILDKIT: 1 - container_name: ark-invest-api-rust-data restart: unless-stopped volumes: - ./data:/ark-invest-api-rust-data/data volumes: data: - Data: diff --git a/src/main.rs b/src/main.rs index e64fec7..ea5fe15 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,70 +1,63 @@ -// use clokwerk::{AsyncScheduler, Job, TimeUnits}; +use clokwerk::{AsyncScheduler, Job, TimeUnits}; // use polars::prelude::LazyFrame; // use polars::prelude::*; -// use std::error::Error; -// use std::result::Result; -// use std::time::Duration; -// use strum::IntoEnumIterator; +use rand::Rng; +use std::error::Error; +use std::result::Result; +use std::time::Duration; +use strum::IntoEnumIterator; mod util; use util::*; -// #[tokio::main] -// async fn main() { -// let mut scheduler = AsyncScheduler::new(); -// scheduler.every(1.day()).at("11:30 pm").run(|| async { -// for x in Ticker::iter() { -// let plan = || -> Result<(), Box> { -// let df = LazyFrame::scan_parquet( -// format!("data/old/{}/part.0.parquet", x), -// ScanArgsParquet::default(), -// )?; -// let df = df_format(x, df)?; -// write_parquet(x, df)?; -// Ok(()) -// }; +#[tokio::main] +async fn main() { + let mut scheduler = AsyncScheduler::new(); + println!("Scheduler Started"); + scheduler.every(1.day()).at("11:30 pm").run(|| async { + for x in Ticker::iter() { + if x == Ticker::ARKVC { + continue; + } + let plan = || -> Result<(), Box> { + let df = Ark::new(Source::Ark, x, None)? + .format()? + .write_parquet()? + .collect()?; -// if plan().is_ok() {} -// } -// }); + println!("{:#?}", df.head(Some(1))); + Ok(()) + }; -// let dfn = read_parquet(Ticker::ARKF).unwrap().collect().unwrap(); -// println!("{:#?}", dfn); -// loop { -// scheduler.run_pending().await; -// // tokio::time::sleep(Duration::from_millis(10)).await; -// tokio::time::sleep(Duration::from_secs(1)).await; -// } -// } + if plan().is_ok() {} + let sec = rand::thread_rng().gen_range(10..=30); + tokio::time::sleep(Duration::from_secs(sec)).await; + } + }); -fn main() -> Result<(), Box> { - // let csv = Ark::merge_old_csv_to_parquet(Ticker::ARKK, None) - // .unwrap() - // .format() - // .unwrap() - // .write_parquet() - // .unwrap() - // .collect() - // .unwrap(); - // println!("{:#?}", csv); - - let read = Ark::new(Source::Read, Ticker::ARKK, None)?.collect()?; - println!("{:#?}", read.dtypes()); - println!("{:#?}", read.get_column_names()); - println!("{:#?}", read); - - // let api = Ark::new(Source::ApiFull, Ticker::ARKK, None) - // .unwrap() - // .collect() - // .unwrap(); - // println!("{:#?}", api); - - // let ark = Ark::new(Source::Ark, Ticker::ARKK, None)?.collect()?; - // println!("{:#?}", ark); - - // let ark = Ark::new(Source::Ark, Ticker::ARKVC, None) - // .unwrap() - // .collect() - // .unwrap(); - // println!("{:#?}", ark); - Ok(()) + loop { + scheduler.run_pending().await; + // tokio::time::sleep(Duration::from_millis(10)).await; + tokio::time::sleep(Duration::from_secs(1)).await; + } } + +// fn main() -> Result<(), Box> { +// let csv = Ark::merge_old_csv_to_parquet(Ticker::ARKK, None)? +// .format()? +// .write_parquet()? +// .collect()?; +// println!("{:#?}", csv); +// let read = Ark::new(Source::Read, Ticker::ARKK, None)?.collect()?; +// println!("{:#?}", read.dtypes()); +// println!("{:#?}", read.get_column_names()); +// println!("{:#?}", read); +// let api = Ark::new(Source::ApiFull, Ticker::ARKK, None)?.collect()?; +// println!("{:#?}", api); + +// let ark = Ark::new(Source::Ark, Ticker::ARKK, None)?.collect()?; +// println!("{:#?}", ark); + +// let ark = Ark::new(Source::Ark, Ticker::ARKVC, None)?.collect()?; +// println!("{:#?}", ark); +// Ok(()) +// } diff --git a/src/util.rs b/src/util.rs index 07795c4..8107be1 100644 --- a/src/util.rs +++ b/src/util.rs @@ -12,7 +12,7 @@ use std::path::Path; use std::result::Result; use strum_macros::EnumIter; -#[derive(strum_macros::Display, EnumIter, Clone, Copy)] +#[derive(strum_macros::Display, EnumIter, Clone, Copy, PartialEq)] pub enum Ticker { ARKVC, ARKF, diff --git a/tests/integration.rs b/tests/integration.rs index 1c75f5e..cab530d 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -15,18 +15,22 @@ fn get_api_arkk() -> Result<(), Box> { .get_api(NaiveDate::from_ymd_opt(2023, 5, 18))? .collect()?; - assert_eq!( - df.get_column_names(), - [ - "company", - "cusip", - "date", - "market_value", - "share_price", - "shares", - "ticker", - "weight" - ] + let expected = [ + "company", + "cusip", + "date", + "market_value", + "share_price", + "shares", + "ticker", + "weight", + "weight_rank", + ]; + let actual = df.get_column_names(); + + assert!( + actual == expected || actual == expected[..expected.len() - 1], + "Column names are wrong" ); Ok(()) }