NexVeridian/wikidata-to-surrealdb
1
0
Fork 0
mirror of https://github.com/NexVeridian/wikidata-to-surrealdb.git synced 2025-09-02 09:59:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

bulk insert and benchmarks

Browse source
This commit is contained in:
Elijah McMorris 2024-01-15 22:17:25 -08:00
parent dc85c7d997
commit 82edfdfbd3
Signed by: NexVeridian
SSH key fingerprint: SHA256:bsA1SKZxuEcEVHAy3gY1HUeM5ykRJl0U0kQHQn0hMg8
12 changed files with 289 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

43
src/main.rs
View file

@ -1,7 +1,6 @@
use anyhow::{Error, Ok, Result};
use indicatif::{ProgressBar, ProgressState, ProgressStyle};
use lazy_static::lazy_static;
use std::{env, fmt::Write, io::BufRead};
use std::{env, io::BufRead};
use surrealdb::{engine::remote::ws::Client, Surreal};
use tokio::time::{sleep, Duration};
mod utils;
@ -16,25 +15,16 @@ lazy_static! {
.expect("THREADED_REQUESTS not set")
.parse()
.expect("Failed to parse THREADED_REQUESTS");
static ref WIKIDATA_BULK_INSERT: bool = env::var("WIKIDATA_BULK_INSERT")
.expect("WIKIDATA_BULK_INSERT not set")
.parse()
.expect("Failed to parse WIKIDATA_BULK_INSERT");
}
#[tokio::main]
async fn main() -> Result<(), Error> {
sleep(Duration::from_secs(10)).await;
let total_size = 113_000_000;
let pb = ProgressBar::new(total_size);
pb.set_style(
ProgressStyle::with_template(
"[{elapsed_precise}] [{wide_bar:.cyan/blue}] {human_pos}/{human_len} ETA:[{eta}]",
)?
.with_key("eta", |state: &ProgressState, w: &mut dyn Write| {
let sec = state.eta().as_secs();
let min = (sec / 60) % 60;
let hr = (sec / 60) / 60;
write!(w, "{}:{:02}:{:02}", hr, min, sec % 60).unwrap()
}),
);
let pb = create_pb().await;
let db = create_db_ws().await?;
let reader = File_Format::new(&WIKIDATA_FILE_FORMAT).reader(&WIKIDATA_FILE_NAME)?;
@ -64,9 +54,26 @@ async fn main() -> Result<(), Error> {
pb.inc(100);
}
}
} else if *WIKIDATA_BULK_INSERT {
create_db_entities_threaded(
None::<Surreal<Client>>,
reader,
Some(pb.clone()),
2500,
100,
CreateVersion::Bulk,
)
.await?;
} else {
create_db_entities_threaded(None::<Surreal<Client>>, reader, Some(pb.clone()), 2500, 100)
.await?;
create_db_entities_threaded(
None::<Surreal<Client>>,
reader,
Some(pb.clone()),
2500,
100,
CreateVersion::Single,
)
.await?;
}
pb.finish();