1.0.0

src/main.rs

@@ -1,38 +1,59 @@
 use clokwerk::{AsyncScheduler, Job, TimeUnits};
-// use polars::prelude::LazyFrame;
-// use polars::prelude::*;
+use futures::future::join_all;
 use rand::Rng;
 use std::error::Error;
 use std::result::Result;
-use std::time::Duration;
+use std::thread;
 use strum::IntoEnumIterator;
+use tokio::task;
+use tokio::time::{sleep, Duration};
 
 mod util;
 use util::*;
+
 #[tokio::main]
 async fn main() {
     let mut scheduler = AsyncScheduler::new();
     println!("Scheduler Started");
-    scheduler.every(1.day()).at("11:30 pm").run(|| async {
-        for x in Ticker::iter() {
-            if x == Ticker::ARKVC {
-                continue;
-            }
-            let plan = || -> Result<(), Box<dyn Error>> {
-                let df = Ark::new(Source::Ark, x, None)?
-                    .format()?
-                    .write_parquet()?
-                    .collect()?;
 
-                println!("{:#?}", df.head(Some(1)));
-                Ok(())
-            };
+    fn ark_plan(ticker: Ticker) -> Result<(), Box<dyn Error>> {
+        println!("Starting: {:#?}", ticker);
+        let sec = Duration::from_secs(rand::thread_rng().gen_range(5 * 60..=30 * 60));
+        // sleep(sec).await;
+        thread::sleep(sec);
 
-            if plan().is_ok() {}
-            let sec = rand::thread_rng().gen_range(10..=30);
-            tokio::time::sleep(Duration::from_secs(sec)).await;
-        }
-    });
+        let df = Ark::new(Source::Ark, ticker, None)?
+            .format()?
+            .write_parquet()?
+            .collect()?;
+
+        println!("Ticker: {:#?}\n{:#?}", ticker, df.tail(Some(1)));
+        Ok(())
+    }
+
+    async fn spawn_ark_plan(ticker: Ticker) -> Result<(), Box<dyn Error + Send>> {
+        task::spawn_blocking(move || ark_plan(ticker).unwrap())
+            .await
+            .unwrap();
+        Ok(())
+    }
+
+    async fn ark_etf() {
+        let futures = Ticker::iter()
+            .filter(|&x| x != Ticker::ARKVC)
+            .map(spawn_ark_plan)
+            .collect::<Vec<_>>();
+
+        join_all(futures).await;
+    }
+
+    // ark_etf().await;
+    scheduler.every(1.day()).at("11:30 pm").run(ark_etf);
+
+    scheduler
+        .every(5.day())
+        .at("11:30 pm")
+        .run(|| async { if spawn_ark_plan(Ticker::ARKVC).await.is_ok() {} });
 
     loop {
         scheduler.run_pending().await;
@@ -40,24 +61,3 @@ async fn main() {
         tokio::time::sleep(Duration::from_secs(1)).await;
     }
 }
-
-// fn main() -> Result<(), Box<dyn std::error::Error>> {
-//     let csv = Ark::merge_old_csv_to_parquet(Ticker::ARKK, None)?
-//         .format()?
-//         .write_parquet()?
-//         .collect()?;
-//     println!("{:#?}", csv);
-//     let read = Ark::new(Source::Read, Ticker::ARKK, None)?.collect()?;
-//     println!("{:#?}", read.dtypes());
-//     println!("{:#?}", read.get_column_names());
-//     println!("{:#?}", read);
-//     let api = Ark::new(Source::ApiFull, Ticker::ARKK, None)?.collect()?;
-//     println!("{:#?}", api);
-
-//     let ark = Ark::new(Source::Ark, Ticker::ARKK, None)?.collect()?;
-//     println!("{:#?}", ark);
-
-//     let ark = Ark::new(Source::Ark, Ticker::ARKVC, None)?.collect()?;
-//     println!("{:#?}", ark);
-//     Ok(())
-// }

src/util.rs

@@ -4,6 +4,8 @@ use polars::datatypes::DataType;
 use polars::lazy::dsl::StrptimeOptions;
 use polars::prelude::*;
 use reqwest::blocking::Client;
+use reqwest::header;
+use reqwest::header::{HeaderMap, HeaderValue};
 use serde_json::Value;
 use std::error::Error;
 use std::fs::{create_dir_all, File};
@@ -12,7 +14,7 @@ use std::path::Path;
 use std::result::Result;
 use strum_macros::EnumIter;
 
-#[derive(strum_macros::Display, EnumIter, Clone, Copy, PartialEq)]
+#[derive(strum_macros::Display, EnumIter, Clone, Copy, PartialEq, Debug)]
 pub enum Ticker {
     ARKVC,
     ARKF,
@@ -408,8 +410,24 @@ pub enum Reader {
 
 impl Reader {
     pub fn get_data_url(&self, url: String) -> Result<LazyFrame, Box<dyn Error>> {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            header::USER_AGENT, 
+            HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"),
+        );
+
+        headers.insert(
+            header::ACCEPT, 
+            HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"),
+        );
+        headers.insert(
+            header::ACCEPT_LANGUAGE,
+            HeaderValue::from_static("en-US,en;q=0.8"),
+        );
+
         let response = Client::builder()
-        .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
+        .default_headers(headers)
+        .gzip(true)
         .build()?.get(url).send()?;
 
         if !response.status().is_success() {