0.4.0

Cargo.toml

@@ -17,12 +17,13 @@ polars = { version = "0.30", features = [
 reqwest = { version = "0.11", features = ["blocking"] }
 glob = { version = "0.3" }
 clokwerk = "0.4.0"
-strum_macros = "0.24"
-strum = "0.24"
+strum_macros = "0.25"
+strum = "0.25"
 tokio = { version = "1.26", features = ["full"] }
 openssl = { version = "0.10", features = ["vendored"] }
 chrono = { version = "0.4", features = ["serde"] }
 serde_json = "1.0"
+rand = "0.8"
 
 [dev-dependencies]
 serial_test = "*"

Dockerfile

@@ -1,15 +1,29 @@
-FROM rust:latest AS builder
+FROM rust:bookworm AS builder
 
-RUN rustup target add x86_64-unknown-linux-musl && apt update && apt install -y musl-tools musl-dev
+RUN apt-get update && \
+	apt install -y  musl-tools musl-dev libssl-dev clang mold
+
+# RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C /usr/local/bin
+# RUN cargo install cargo-nextest --locked
 
 WORKDIR /ark-invest-api-rust-data
 
 COPY . .
 
+RUN rustup target add x86_64-unknown-linux-musl && rustup update && cargo update
+
 RUN --mount=type=cache,target=/usr/local/cargo,from=rust,source=/usr/local/cargo \
 	--mount=type=cache,target=./target \
 	cargo build --target x86_64-unknown-linux-musl --release && \
-	cp ./target/x86_64-unknown-linux-musl/release/ark-invest-api-rust-data .
+	cp ./target/target/x86_64-unknown-linux-musl/release/ark-invest-api-rust-data .
+
+FROM builder AS test
+
+RUN --mount=type=cache,target=/usr/local/cargo,from=rust,source=/usr/local/cargo \
+	--mount=type=cache,target=./target \
+	cargo nextest run --release --target x86_64-unknown-linux-musl \
+	-E "all() - test(get_api) - kind(bin)"
 
 FROM alpine:latest
 

docker-compose.yml

@@ -3,15 +3,15 @@ services:
     ark-invest-api-rust-data:
         # image: ghcr.io/NexVeridian/ark-invest-api-rust-data:latest 
         image: ark-invest-api-rust-data
+        container_name: ark-invest-api-rust-data
         build:
             context: .
+            target: test
             args:
                 DOCKER_BUILDKIT: 1
-        container_name: ark-invest-api-rust-data
         restart: unless-stopped
         volumes:
             - ./data:/ark-invest-api-rust-data/data
 
 volumes:
     data:
-    Data:

src/main.rs

@@ -1,70 +1,63 @@
-// use clokwerk::{AsyncScheduler, Job, TimeUnits};
+use clokwerk::{AsyncScheduler, Job, TimeUnits};
 // use polars::prelude::LazyFrame;
 // use polars::prelude::*;
-// use std::error::Error;
-// use std::result::Result;
-// use std::time::Duration;
-// use strum::IntoEnumIterator;
+use rand::Rng;
+use std::error::Error;
+use std::result::Result;
+use std::time::Duration;
+use strum::IntoEnumIterator;
 
 mod util;
 use util::*;
-// #[tokio::main]
-// async fn main() {
-//     let mut scheduler = AsyncScheduler::new();
-//     scheduler.every(1.day()).at("11:30 pm").run(|| async {
-//         for x in Ticker::iter() {
-//             let plan = || -> Result<(), Box<dyn Error>> {
-//                 let df = LazyFrame::scan_parquet(
-//                     format!("data/old/{}/part.0.parquet", x),
-//                     ScanArgsParquet::default(),
-//                 )?;
-//                 let df = df_format(x, df)?;
-//                 write_parquet(x, df)?;
-//                 Ok(())
-//             };
+#[tokio::main]
+async fn main() {
+    let mut scheduler = AsyncScheduler::new();
+    println!("Scheduler Started");
+    scheduler.every(1.day()).at("11:30 pm").run(|| async {
+        for x in Ticker::iter() {
+            if x == Ticker::ARKVC {
+                continue;
+            }
+            let plan = || -> Result<(), Box<dyn Error>> {
+                let df = Ark::new(Source::Ark, x, None)?
+                    .format()?
+                    .write_parquet()?
+                    .collect()?;
 
-//             if plan().is_ok() {}
-//         }
-//     });
-
-//     let dfn = read_parquet(Ticker::ARKF).unwrap().collect().unwrap();
-//     println!("{:#?}", dfn);
-//     loop {
-//         scheduler.run_pending().await;
-//         // tokio::time::sleep(Duration::from_millis(10)).await;
-//         tokio::time::sleep(Duration::from_secs(1)).await;
-//     }
-// }
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // let csv = Ark::merge_old_csv_to_parquet(Ticker::ARKK, None)
-    //     .unwrap()
-    //     .format()
-    //     .unwrap()
-    //     .write_parquet()
-    //     .unwrap()
-    //     .collect()
-    //     .unwrap();
-    // println!("{:#?}", csv);
-
-    let read = Ark::new(Source::Read, Ticker::ARKK, None)?.collect()?;
-    println!("{:#?}", read.dtypes());
-    println!("{:#?}", read.get_column_names());
-    println!("{:#?}", read);
-
-    // let api = Ark::new(Source::ApiFull, Ticker::ARKK, None)
-    //     .unwrap()
-    //     .collect()
-    //     .unwrap();
-    // println!("{:#?}", api);
-
-    // let ark = Ark::new(Source::Ark, Ticker::ARKK, None)?.collect()?;
-    // println!("{:#?}", ark);
-
-    // let ark = Ark::new(Source::Ark, Ticker::ARKVC, None)
-    //     .unwrap()
-    //     .collect()
-    //     .unwrap();
-    // println!("{:#?}", ark);
+                println!("{:#?}", df.head(Some(1)));
                 Ok(())
+            };
+
+            if plan().is_ok() {}
+            let sec = rand::thread_rng().gen_range(10..=30);
+            tokio::time::sleep(Duration::from_secs(sec)).await;
+        }
+    });
+
+    loop {
+        scheduler.run_pending().await;
+        // tokio::time::sleep(Duration::from_millis(10)).await;
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
 }
+
+// fn main() -> Result<(), Box<dyn std::error::Error>> {
+//     let csv = Ark::merge_old_csv_to_parquet(Ticker::ARKK, None)?
+//         .format()?
+//         .write_parquet()?
+//         .collect()?;
+//     println!("{:#?}", csv);
+//     let read = Ark::new(Source::Read, Ticker::ARKK, None)?.collect()?;
+//     println!("{:#?}", read.dtypes());
+//     println!("{:#?}", read.get_column_names());
+//     println!("{:#?}", read);
+//     let api = Ark::new(Source::ApiFull, Ticker::ARKK, None)?.collect()?;
+//     println!("{:#?}", api);
+
+//     let ark = Ark::new(Source::Ark, Ticker::ARKK, None)?.collect()?;
+//     println!("{:#?}", ark);
+
+//     let ark = Ark::new(Source::Ark, Ticker::ARKVC, None)?.collect()?;
+//     println!("{:#?}", ark);
+//     Ok(())
+// }

src/util.rs

@@ -12,7 +12,7 @@ use std::path::Path;
 use std::result::Result;
 use strum_macros::EnumIter;
 
-#[derive(strum_macros::Display, EnumIter, Clone, Copy)]
+#[derive(strum_macros::Display, EnumIter, Clone, Copy, PartialEq)]
 pub enum Ticker {
     ARKVC,
     ARKF,

tests/integration.rs

@@ -15,9 +15,7 @@ fn get_api_arkk() -> Result<(), Box<dyn Error>> {
     .get_api(NaiveDate::from_ymd_opt(2023, 5, 18))?
     .collect()?;
 
-    assert_eq!(
-        df.get_column_names(),
-        [
+    let expected = [
         "company",
         "cusip",
         "date",
@@ -25,8 +23,14 @@ fn get_api_arkk() -> Result<(), Box<dyn Error>> {
         "share_price",
         "shares",
         "ticker",
-            "weight"
-        ]
+        "weight",
+        "weight_rank",
+    ];
+    let actual = df.get_column_names();
+
+    assert!(
+        actual == expected || actual == expected[..expected.len() - 1],
+        "Column names are wrong"
     );
     Ok(())
 }