NexVeridian/ark-invest-api-rust-data
1
0
Fork 0
mirror of https://github.com/NexVeridian/ark-invest-api-rust-data.git synced 2025-09-02 01:49:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: more renames

Browse source
This commit is contained in:
Elijah McMorris 2025-02-03 12:47:44 -08:00
parent 5aab89ff0d
commit 051d18494b
Signed by: NexVeridian
SSH key fingerprint: SHA256:bsA1SKZxuEcEVHAy3gY1HUeM5ykRJl0U0kQHQn0hMg8
12 changed files with 441 additions and 322 deletions
Download patch file Download diff file Expand all files Collapse all files

12
.cargo/config.toml
View file

@ -8,11 +8,11 @@ t = "nextest run"
[build] [build]
target-dir = "target/target" target-dir = "target/target"
[unstable] # [unstable]
codegen-backend = true # codegen-backend = true
[profile.release] # [profile.release]
codegen-backend = "cranelift" # codegen-backend = "cranelift"
[profile.dev] # [profile.dev]
codegen-backend = "cranelift" # codegen-backend = "cranelift"

517
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

12
Cargo.toml
View file

@ -18,17 +18,17 @@ polars = { version = "0.32", features = [
reqwest = { version = "0.12", features = ["blocking", "gzip"] } reqwest = { version = "0.12", features = ["blocking", "gzip"] }
glob = { version = "0.3" } glob = { version = "0.3" }
clokwerk = "0.4" clokwerk = "0.4"
strum_macros = "0.26" strum_macros = "0.27"
strum = "0.26" strum = "0.27"
tokio = { version = "1.34", features = ["full"] } tokio = { version = "1.43", features = ["full"] }
openssl = { version = "0.10", features = ["vendored"] } openssl = { version = "0.10", features = ["vendored"] }
chrono = { version = "0.4", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] }
serde_json = "1.0" serde_json = "1.0"
rand = "0.8" rand = "0.9"
futures = "0.3" futures = "0.3"
lazy_static = "1.4" lazy_static = "1.5"
anyhow = "1.0" anyhow = "1.0"
[dev-dependencies] [dev-dependencies]
rstest = "0.23" rstest = "0.24"
pretty_assertions = "1.4" pretty_assertions = "1.4"

30
flake.lock generated
View file

@ -3,11 +3,11 @@
"advisory-db": { "advisory-db": {
"flake": false, "flake": false,
"locked": { "locked": {
"lastModified": 1738539423, "lastModified": 1740407442,
"narHash": "sha256-qb4FLJFuIHdzI1oeVor69678RugKy8YwsLRQd38fJnc=", "narHash": "sha256-EGzWKm5cUDDJbwVzxSB4N/+CIVycwOG60Gh5f1Vp7JM=",
"owner": "rustsec", "owner": "rustsec",
"repo": "advisory-db", "repo": "advisory-db",
"rev": "08617accdc251d22bc8fd4e3bd62cf53eeddf611", "rev": "2e25d9665f10de885c81a9fb9d51a289f625b05f",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -18,11 +18,11 @@
}, },
"crane": { "crane": {
"locked": { "locked": {
"lastModified": 1737689766, "lastModified": 1739936662,
"narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=", "narHash": "sha256-x4syUjNUuRblR07nDPeLDP7DpphaBVbUaSoeZkFbGSk=",
"owner": "ipetkov", "owner": "ipetkov",
"repo": "crane", "repo": "crane",
"rev": "6fe74265bbb6d016d663b1091f015e2976c4a527", "rev": "19de14aaeb869287647d9461cbd389187d8ecdb7",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -39,11 +39,11 @@
"rust-analyzer-src": "rust-analyzer-src" "rust-analyzer-src": "rust-analyzer-src"
}, },
"locked": { "locked": {
"lastModified": 1738564312, "lastModified": 1740897281,
"narHash": "sha256-awAp1Qe+c95PQxP7v+Zfse+w3URaP3UQLCRlaPMzYtE=", "narHash": "sha256-lUCeMKV8sUn8Y4R+4OzFMqH0k00KqvbIu0vSIMsWpH0=",
"owner": "nix-community", "owner": "nix-community",
"repo": "fenix", "repo": "fenix",
"rev": "d99d2a562b9c9d5f0e4399e5bb677b37a791c7eb", "rev": "2a4bbf46e00e06967f55b3a55a685d2039f2b572",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -72,11 +72,11 @@
}, },
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1738517419, "lastModified": 1740791350,
"narHash": "sha256-tttEXgKimgbtPvxFl+Avos4P4lssIqxHhxpLbbvNekk=", "narHash": "sha256-igS2Z4tVw5W/x3lCZeeadt0vcU9fxtetZ/RyrqsCRQ0=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "8532db2a88ba56de9188af72134d93e39fd825f3", "rev": "199169a2135e6b864a888e89a2ace345703c025d",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -98,11 +98,11 @@
"rust-analyzer-src": { "rust-analyzer-src": {
"flake": false, "flake": false,
"locked": { "locked": {
"lastModified": 1738488035, "lastModified": 1740851167,
"narHash": "sha256-sLLW0S7OGlFYgNvAQnqWK1Ws5V1YNGvfXHdWoZ91CeI=", "narHash": "sha256-sFRsGYrI2Z4X+P9jkvb7XZJG6DImVhz8BSELBT2mLdo=",
"owner": "rust-lang", "owner": "rust-lang",
"repo": "rust-analyzer", "repo": "rust-analyzer",
"rev": "f3998f7f8a197596c5edf72e937996e6674b423b", "rev": "76567ee28dacac2636aeb6e5b6a9560ffca21d87",
"type": "github" "type": "github"
}, },
"original": { "original": {

8
flake.nix
View file

@ -44,7 +44,6 @@
fpc.rustc fpc.rustc
fpc.clippy fpc.clippy
fpc.rust-src fpc.rust-src
fpc.rustc
fpc.rustfmt fpc.rustfmt
fpc.rustc-codegen-cranelift-preview fpc.rustc-codegen-cranelift-preview
]) ])
@ -59,18 +58,23 @@
buildInputs = buildInputs =
[ [
pkgs.openssl
pkgs.pkg-config pkgs.pkg-config
pkgs.openssl
pkgs.openssl.dev
# Add additional build inputs here # Add additional build inputs here
] ]
++ lib.optionals pkgs.stdenv.isDarwin [ ++ lib.optionals pkgs.stdenv.isDarwin [
# Additional darwin specific inputs can be set here # Additional darwin specific inputs can be set here
pkgs.libiconv pkgs.libiconv
pkgs.darwin.apple_sdk.frameworks.SystemConfiguration pkgs.darwin.apple_sdk.frameworks.SystemConfiguration
pkgs.darwin.apple_sdk.frameworks.Security
]; ];
# Additional environment variables can be set directly # Additional environment variables can be set directly
# MY_CUSTOM_VAR = "some value"; # MY_CUSTOM_VAR = "some value";
OPENSSL_NO_VENDOR = "1";
OPENSSL_DIR = "${pkgs.openssl.dev}";
OPENSSL_LIB_DIR = "${pkgs.openssl.out}/lib";
}; };
craneLibLLvmTools = craneLib.overrideToolchain ( craneLibLLvmTools = craneLib.overrideToolchain (

25
justfile
View file

@ -1,6 +1,23 @@
check:
just precommit-shared
nix flake update
nix flake check -v
precommit: precommit:
cargo update just precommit-shared
cargo check cargo check
cargo fmt just test
cargo t
cargo clippy --fix --allow-dirty alias t := test
test:
cargo t --no-fail-fast
precommit-shared:
cargo upgrade -v
cargo update
cargo fmt --all
just clippy
clippy:
cargo clippy --all --fix --allow-dirty -- -W clippy::nursery -W rust-2018-idioms \
-A clippy::future_not_send -A clippy::option_if_let_else -A clippy::or_fun_call

2
src/main.rs
View file

@ -47,7 +47,7 @@ fn csv_merge() -> Result<(), Error> {
fn ark_plan(ticker: Ticker) -> Result<(), Error> { fn ark_plan(ticker: Ticker) -> Result<(), Error> {
println!("Starting: {:#?}", ticker); println!("Starting: {:#?}", ticker);
let sec = Duration::from_secs(rand::thread_rng().gen_range(30 * 60..=4 * 60 * 60)); let sec = Duration::from_secs(rand::rng().random_range(30 * 60..=4 * 60 * 60));
// sleep(sec).await; // sleep(sec).await;
thread::sleep(sec); thread::sleep(sec);

27
src/util.rs
View file

@ -16,7 +16,7 @@ pub mod df;
mod format; mod format;
pub mod ticker; pub mod ticker;
#[derive(Debug, Default, EnumString, Clone, Copy, PartialEq)] #[derive(Debug, Default, EnumString, Clone, Copy, PartialEq, Eq)]
pub enum Source { pub enum Source {
// Reads Parquet file if exists // Reads Parquet file if exists
Read, Read,
@ -132,12 +132,7 @@ impl Ark {
fn concat_df(dfs: Vec<DF>) -> Result<DF, Error> { fn concat_df(dfs: Vec<DF>) -> Result<DF, Error> {
// with dedupe // with dedupe
let df = concat( let df = concat(dfs.lazy(), Default::default())?;
dfs.lazy(),
UnionArgs {
..Default::default()
},
)?;
Self::dedupe(df.into()) Self::dedupe(df.into())
} }
@ -378,6 +373,10 @@ impl Ark {
.str() .str()
.replace_all(lit("."), lit(""), true) .replace_all(lit("."), lit(""), true)
.str() .str()
.replace_all(lit(" &CURITY"), lit(""), true)
.str()
.replace_all(lit(" &"), lit(""), true)
.str()
.replace(lit("HLDGS"), lit(""), true) .replace(lit("HLDGS"), lit(""), true)
.str() .str()
.replace(lit("HOLDINGS"), lit(""), true) .replace(lit("HOLDINGS"), lit(""), true)
@ -394,6 +393,8 @@ impl Ark {
.str() .str()
.replace(lit(" ADR"), lit(""), true) .replace(lit(" ADR"), lit(""), true)
.str() .str()
.replace(lit("DR"), lit(""), true)
.str()
.replace(lit(" SA"), lit(""), true) .replace(lit(" SA"), lit(""), true)
.str() .str()
.replace(lit(" NV"), lit(""), true) .replace(lit(" NV"), lit(""), true)
@ -561,13 +562,7 @@ impl Ark {
dfs.push(LazyCsvReader::new(x).finish()?); dfs.push(LazyCsvReader::new(x).finish()?);
} }
let mut df = concat( let mut df = concat(dfs, Default::default())?.into();
dfs,
UnionArgs {
..Default::default()
},
)?
.into();
if Self::read_parquet(&ticker, path.as_ref()).is_ok() { if Self::read_parquet(&ticker, path.as_ref()).is_ok() {
let df_old = Self::read_parquet(&ticker, path.as_ref())?; let df_old = Self::read_parquet(&ticker, path.as_ref())?;
@ -611,7 +606,7 @@ mod tests {
], ],
)?; )?;
Ark::write_df_parquet("data/test/ARKW.parquet".into(), test_df.clone().into())?; Ark::write_df_parquet("data/test/ARKW.parquet".into(), test_df.into())?;
let read = Ark::new(Source::Read, Ticker::ARKW, Some("data/test".to_owned()))?.collect()?; let read = Ark::new(Source::Read, Ticker::ARKW, Some("data/test".to_owned()))?.collect()?;
fs::remove_file("data/test/ARKW.parquet")?; fs::remove_file("data/test/ARKW.parquet")?;
@ -637,7 +632,7 @@ mod tests {
Some("ARKB"), Some("ARKB"),
], ],
)?; )?;
Ark::write_df_parquet("data/test/ARKF.parquet".into(), test_df.clone().into())?; Ark::write_df_parquet("data/test/ARKF.parquet".into(), test_df.into())?;
let read = Ark::new(Source::Read, Ticker::ARKF, Some("data/test".to_owned()))?.collect()?; let read = Ark::new(Source::Read, Ticker::ARKF, Some("data/test".to_owned()))?.collect()?;
fs::remove_file("data/test/ARKF.parquet")?; fs::remove_file("data/test/ARKF.parquet")?;

2
src/util/data_reader.rs
View file

@ -6,7 +6,7 @@ use reqwest::blocking::Client;
use serde_json::Value; use serde_json::Value;
use std::io::Cursor; use std::io::Cursor;
#[derive(Debug, Clone, Copy, PartialEq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Reader { pub enum Reader {
Csv, Csv,
Json, Json,

12
src/util/df.rs
View file

@ -10,27 +10,27 @@ pub enum DF {
impl From<LazyFrame> for DF { impl From<LazyFrame> for DF {
fn from(lf: LazyFrame) -> Self { fn from(lf: LazyFrame) -> Self {
DF::LazyFrame(lf) Self::LazyFrame(lf)
} }
} }
impl From<DataFrame> for DF { impl From<DataFrame> for DF {
fn from(df: DataFrame) -> Self { fn from(df: DataFrame) -> Self {
DF::DataFrame(df) Self::DataFrame(df)
} }
} }
impl DF { impl DF {
pub fn collect(self) -> anyhow::Result<DataFrame, Error> { pub fn collect(self) -> anyhow::Result<DataFrame, Error> {
match self { match self {
DF::LazyFrame(x) => Ok(x.collect()?), Self::LazyFrame(x) => Ok(x.collect()?),
DF::DataFrame(x) => Ok(x), Self::DataFrame(x) => Ok(x),
} }
} }
pub fn lazy(self) -> LazyFrame { pub fn lazy(self) -> LazyFrame {
match self { match self {
DF::LazyFrame(x) => x, Self::LazyFrame(x) => x,
DF::DataFrame(x) => x.lazy(), Self::DataFrame(x) => x.lazy(),
} }
} }
} }

12
src/util/format/ticker.rs
View file

@ -6,7 +6,7 @@ use strum_macros::EnumIter;
use crate::util::df::DF; use crate::util::df::DF;
#[allow(clippy::upper_case_acronyms, non_camel_case_types)] #[allow(clippy::upper_case_acronyms, non_camel_case_types)]
#[derive(Debug, strum_macros::Display, EnumIter, Clone, Copy, PartialEq)] #[derive(Debug, strum_macros::Display, EnumIter, Clone, Copy, PartialEq, Eq)]
pub enum Ticker { pub enum Ticker {
ARKW, ARKW,
MKFG, MKFG,
@ -16,7 +16,7 @@ pub enum Ticker {
impl Ticker { impl Ticker {
pub fn all(mut df: DF) -> Result<DF, Error> { pub fn all(mut df: DF) -> Result<DF, Error> {
for ticker in Ticker::iter() { for ticker in Self::iter() {
df = ticker.format(df)?; df = ticker.format(df)?;
} }
Ok(df) Ok(df)
@ -24,10 +24,10 @@ impl Ticker {
pub fn format(&self, df: DF) -> Result<DF, Error> { pub fn format(&self, df: DF) -> Result<DF, Error> {
match self { match self {
Ticker::ARKW => Self::arkw(df), Self::ARKW => Self::arkw(df),
Ticker::MKFG => Self::mkfg(df), Self::MKFG => Self::mkfg(df),
Ticker::XYZ => Self::xyz(df), Self::XYZ => Self::xyz(df),
Ticker::CASH_USD => Self::cash_usd(df), Self::CASH_USD => Self::cash_usd(df),
} }
} }

102
src/util/ticker.rs
View file

@ -1,6 +1,6 @@
use strum_macros::EnumIter; use strum_macros::EnumIter;
#[derive(Clone, Copy, PartialEq)] #[derive(Clone, Copy, PartialEq, Eq)]
pub enum DataSource { pub enum DataSource {
ArkVenture, ArkVenture,
Ark, Ark,
@ -10,7 +10,7 @@ pub enum DataSource {
} }
#[allow(clippy::upper_case_acronyms, non_camel_case_types)] #[allow(clippy::upper_case_acronyms, non_camel_case_types)]
#[derive(Debug, Default, strum_macros::Display, EnumIter, Clone, Copy, PartialEq)] #[derive(Debug, Default, strum_macros::Display, EnumIter, Clone, Copy, PartialEq, Eq)]
pub enum Ticker { pub enum Ticker {
ARKVX, ARKVX,
@ -46,72 +46,64 @@ pub enum Ticker {
} }
impl Ticker { impl Ticker {
pub fn value(&self) -> &str { pub const fn value(&self) -> &str {
match *self { match *self {
Ticker::ARKVX => "ARK_VENTURE_FUND_ARKVX_HOLDINGS.csv", Self::ARKVX => "ARK_VENTURE_FUND_ARKVX_HOLDINGS.csv",
Ticker::ARKF => "FINTECH_INNOVATION", Self::ARKF => "FINTECH_INNOVATION",
Ticker::ARKG => "GENOMIC_REVOLUTION", Self::ARKG => "GENOMIC_REVOLUTION",
Ticker::ARKK => "INNOVATION", Self::ARKK => "INNOVATION",
Ticker::ARKQ => "AUTONOMOUS_TECH._&_ROBOTICS", Self::ARKQ => "AUTONOMOUS_TECH._&_ROBOTICS",
Ticker::ARKW => "NEXT_GENERATION_INTERNET", Self::ARKW => "NEXT_GENERATION_INTERNET",
Ticker::ARKX => "SPACE_EXPLORATION_&_INNOVATION", Self::ARKX => "SPACE_EXPLORATION_&_INNOVATION",
Ticker::ARKA => "ARKA", Self::ARKA => "ARKA",
Ticker::ARKZ => "ARKZ", Self::ARKZ => "ARKZ",
Ticker::ARKC => "ARKC", Self::ARKC => "ARKC",
Ticker::ARKD => "ARKD", Self::ARKD => "ARKD",
Ticker::ARKY => "ARKY", Self::ARKY => "ARKY",
Ticker::ARKB => "21SHARES_BITCOIN", Self::ARKB => "21SHARES_BITCOIN",
Ticker::PRNT => "THE_3D_PRINTING", Self::PRNT => "THE_3D_PRINTING",
Ticker::IZRL => "ISRAEL_INNOVATIVE_TECHNOLOGY", Self::IZRL => "ISRAEL_INNOVATIVE_TECHNOLOGY",
Ticker::EUROPE_ARKI => "artificial-intelligence-robotics", Self::EUROPE_ARKI => "artificial-intelligence-robotics",
Ticker::EUROPE_ARKG => "genomic-revolution", Self::EUROPE_ARKG => "genomic-revolution",
Ticker::EUROPE_ARKK => "innovation", Self::EUROPE_ARKK => "innovation",
Ticker::CYBR => "cybersecurity-and-data-privacy", Self::CYBR => "cybersecurity-and-data-privacy",
Ticker::CYCL => "circular-economy-enablers", Self::CYCL => "circular-economy-enablers",
Ticker::FOOD => "sustainable-future-of-food", Self::FOOD => "sustainable-future-of-food",
Ticker::LIFE => "environmental-impact-100", Self::LIFE => "environmental-impact-100",
Ticker::LUSA => "usa-environmental-impact", Self::LUSA => "usa-environmental-impact",
Ticker::NFRA => "global-sustainable-infrastructure", Self::NFRA => "global-sustainable-infrastructure",
Ticker::PMNT => "digital-payments-economy", Self::PMNT => "digital-payments-economy",
} }
} }
pub fn data_source(&self) -> DataSource { pub const fn data_source(&self) -> DataSource {
match *self { match *self {
Ticker::ARKVX => DataSource::ArkVenture, Self::ARKVX => DataSource::ArkVenture,
Ticker::ARKF Self::ARKF | Self::ARKG | Self::ARKK | Self::ARKQ | Self::ARKW | Self::ARKX => {
| Ticker::ARKG DataSource::Ark
| Ticker::ARKK
| Ticker::ARKQ
| Ticker::ARKW
| Ticker::ARKX => DataSource::Ark,
Ticker::ARKA
| Ticker::ARKZ
| Ticker::ARKC
| Ticker::ARKD
| Ticker::ARKY
| Ticker::ARKB => DataSource::Shares21,
Ticker::PRNT | Ticker::IZRL => DataSource::Ark,
Ticker::EUROPE_ARKI | Ticker::EUROPE_ARKG | Ticker::EUROPE_ARKK => {
DataSource::ArkEurope
} }
Ticker::CYBR Self::ARKA | Self::ARKZ | Self::ARKC | Self::ARKD | Self::ARKY | Self::ARKB => {
| Ticker::CYCL DataSource::Shares21
| Ticker::FOOD }
| Ticker::LIFE
| Ticker::LUSA Self::PRNT | Self::IZRL => DataSource::Ark,
| Ticker::NFRA
| Ticker::PMNT => DataSource::Rize, Self::EUROPE_ARKI | Self::EUROPE_ARKG | Self::EUROPE_ARKK => DataSource::ArkEurope,
Self::CYBR
| Self::CYCL
| Self::FOOD
| Self::LIFE
| Self::LUSA
| Self::NFRA
| Self::PMNT => DataSource::Rize,
} }
} }