NexVeridian/ark-invest-api-rust-data
1
0
Fork 0
mirror of https://github.com/NexVeridian/ark-invest-api-rust-data.git synced 2025-09-02 01:49:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

0.2.1

Browse source
This commit is contained in:
Elijah McMorris 2023-06-07 02:23:14 +00:00
parent be08848b0d
commit 4074a97ae2
Signed by: NexVeridian
SSH key fingerprint: SHA256:bsA1SKZxuEcEVHAy3gY1HUeM5ykRJl0U0kQHQn0hMg8
2 changed files with 135 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/main.rs
View file

@ -8,7 +8,6 @@
mod util; mod util;
use util::*; use util::*;
// #[tokio::main] // #[tokio::main]
// async fn main() { // async fn main() {
// let mut scheduler = AsyncScheduler::new(); // let mut scheduler = AsyncScheduler::new();
@ -38,13 +37,13 @@ use util::*;
// } // }
fn main() { fn main() {
let dfn = df_format(Ticker::ARKF, read_parquet(Ticker::ARKF).unwrap()).unwrap(); let dfn = df_format(read_parquet(Ticker::ARKVC).unwrap()).unwrap();
println!("{:#?}", dfn); println!("{:#?}", dfn);
// update_parquet(Ticker::ARKF).unwrap(); // update_parquet(Ticker::ARKVC).unwrap();
let update = df_format(Ticker::ARKF, get_csv(Ticker::ARKF).unwrap()).unwrap(); // let update = df_format(Ticker::ARKF, get_csv(Ticker::ARKF).unwrap()).unwrap();
// let update = get_csv(Ticker::ARKF).unwrap().collect().unwrap(); // let update = get_csv(Ticker::ARKF).unwrap().collect().unwrap();
// let x = read_parquet(Ticker::ARKF).unwrap().collect().unwrap(); // let x = df_format(read_parquet(Ticker::ARKVC).unwrap()).unwrap();
println!("{:#?}", update); // println!("{:#?}", x);
} }

81
src/util.rs
View file

@ -44,7 +44,7 @@ pub fn merge_csv_to_parquet(folder: Ticker) -> Result<(), Box<dyn Error>> {
let df = concat(dfs, false, true)?; let df = concat(dfs, false, true)?;
write_parquet(folder, df_format(folder, df)?)?; write_parquet(folder, df_format(df)?)?;
Ok(()) Ok(())
} }
@ -53,7 +53,12 @@ pub fn update_parquet(ticker: Ticker) -> Result<(), Box<dyn Error>> {
let mut df = read_parquet(ticker)?; let mut df = read_parquet(ticker)?;
df = concat(vec![df, update], false, true)?.unique_stable(None, UniqueKeepStrategy::First); df = concat(
vec![df_format(df)?.lazy(), df_format(update)?.lazy()],
false,
true,
)?
.unique_stable(None, UniqueKeepStrategy::First);
write_parquet(ticker, df.collect()?)?; write_parquet(ticker, df.collect()?)?;
Ok(()) Ok(())
@ -74,40 +79,40 @@ pub fn write_parquet(ticker: Ticker, mut df: DataFrame) -> Result<(), Box<dyn Er
Ok(()) Ok(())
} }
pub fn df_format(ticker: Ticker, mut dfl: LazyFrame) -> Result<DataFrame, Box<dyn Error>> { pub fn df_format(df: LazyFrame) -> Result<DataFrame, Box<dyn Error>> {
match ticker { let mut df = df.collect()?;
Ticker::ARKVC => {
dfl = dfl.rename(vec!["CUSIP", "weight (%)"], vec!["cusip", "weight"]);
let df = dfl if df.get_column_names().contains(&"market_value_($)") {
.with_columns(vec![ df = df
col("date").str().strptime(StrpTimeOptions { .lazy()
date_dtype: DataType::Date, .rename(
fmt: Some("%m/%d/%Y".into()), vec!["market_value_($), weight_(%)"],
strict: false, vec!["market_value, weight"],
exact: true, )
cache: false, .collect()?;
tz_aware: false, }
utc: false, if df.get_column_names().contains(&"market value ($)") {
}), df = df
col("weight") .lazy()
.str() .rename(
.extract(r"[0-9]*\.[0-9]+", 0) vec!["market value ($), weight (%)"],
.cast(DataType::Float64), vec!["market_value, weight"],
]) )
.filter(col("date").is_not_null()) .collect()?;
}
if df.get_column_names().contains(&"CUSIP") {
df = df
.lazy()
.rename(vec!["CUSIP", "weight (%)"], vec!["cusip", "weight"])
.collect()?; .collect()?;
Ok(df)
} }
_ => {
let mut df = dfl.collect()?;
let col_names = df.get_column_names();
if df.rename("market_value_($)", "market_value").is_ok() {} // if df.rename("market_value_($)", "market_value").is_ok() {}
if df.rename("market value ($)", "market_value").is_ok() {} // if df.rename("market value ($)", "market_value").is_ok() {}
if df.rename("weight_(%)", "weight").is_ok() {} // if df.rename("weight_(%)", "weight").is_ok() {}
if df.rename("weight (%)", "weight").is_ok() {} // if df.rename("weight (%)", "weight").is_ok() {}
// if df.rename("CUSIP", "cusip").is_ok() {}
if df.get_column_names().contains(&"fund") { if df.get_column_names().contains(&"fund") {
_ = df.drop_in_place("fund"); _ = df.drop_in_place("fund");
} }
@ -168,7 +173,9 @@ pub fn df_format(ticker: Ticker, mut dfl: LazyFrame) -> Result<DataFrame, Box<dy
.filter(col("date").is_not_null()) .filter(col("date").is_not_null())
.collect()?; .collect()?;
if !df.get_column_names().contains(&"share_price") { if !df.get_column_names().contains(&"share_price")
&& df.get_column_names().contains(&"market_value")
{
df = df df = df
.lazy() .lazy()
.with_column( .with_column(
@ -178,9 +185,10 @@ pub fn df_format(ticker: Ticker, mut dfl: LazyFrame) -> Result<DataFrame, Box<dy
.cast(DataType::Float64) .cast(DataType::Float64)
.round(2), .round(2),
) )
.collect()?; .collect()?
} }
if df.get_column_names().contains(&"share_price") {
df = df.select([ df = df.select([
"date", "date",
"ticker", "ticker",
@ -191,11 +199,12 @@ pub fn df_format(ticker: Ticker, mut dfl: LazyFrame) -> Result<DataFrame, Box<dy
"share_price", "share_price",
"weight", "weight",
])?; ])?;
} else {
df = df.select(["date", "ticker", "cusip", "company", "weight"])?;
}
Ok(df) Ok(df)
} }
}
}
pub fn get_csv(ticker: Ticker) -> Result<LazyFrame, Box<dyn Error>> { pub fn get_csv(ticker: Ticker) -> Result<LazyFrame, Box<dyn Error>> {
let url = match ticker { let url = match ticker {