NexVeridian/wikidata-to-surrealdb
1
0
Fork 0
mirror of https://github.com/NexVeridian/wikidata-to-surrealdb.git synced 2025-09-02 09:59:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix bulk insert

Browse source
This commit is contained in:
Elijah McMorris 2024-08-25 21:23:10 -07:00
parent d508518beb
commit 64d13fff58
Signed by: NexVeridian
SSH key fingerprint: SHA256:bsA1SKZxuEcEVHAy3gY1HUeM5ykRJl0U0kQHQn0hMg8
1 changed files with 13 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/utils.rs
View file

@ -1,5 +1,6 @@
use anyhow::{Error, Result}; use anyhow::{Error, Result};
use bzip2::read::MultiBzDecoder; use bzip2::read::MultiBzDecoder;
use core::panic;
use futures::future::join_all; use futures::future::join_all;
use indicatif::{ProgressBar, ProgressState, ProgressStyle}; use indicatif::{ProgressBar, ProgressState, ProgressStyle};
use lazy_static::lazy_static; use lazy_static::lazy_static;
@ -146,17 +147,17 @@ pub async fn create_db_entities_bulk(
} }
} }
db.query("insert into Entity ($data_vec) RETURN NONE;") db.insert::<Vec<EntityMini>>("Entity")
.bind(("data_vec", data_vec)) .content(data_vec)
.await?; .await?;
db.query("insert into Claims ($claims_vec) RETURN NONE;") db.insert::<Vec<Claims>>("Claims")
.bind(("claims_vec", claims_vec)) .content(claims_vec)
.await?; .await?;
db.query("insert into Property ($property_vec) RETURN NONE;") db.insert::<Vec<EntityMini>>("Property")
.bind(("property_vec", property_vec)) .content(property_vec)
.await?; .await?;
db.query("insert into Lexeme ($lexeme_vec) RETURN NONE;") db.insert::<Vec<EntityMini>>("Lexeme")
.bind(("lexeme_vec", lexeme_vec)) .content(lexeme_vec)
.await?; .await?;
if let Some(ref p) = pb { if let Some(ref p) = pb {
@ -172,7 +173,7 @@ pub async fn create_db_entities_bulk_filter(
batch_size: usize, batch_size: usize,
) -> Result<(), Error> { ) -> Result<(), Error> {
let db_mem = create_db_mem().await?; let db_mem = create_db_mem().await?;
create_db_entities_bulk(&db_mem, lines, pb, batch_size).await?; create_db_entities_bulk(&db_mem, lines, &None, batch_size).await?;
let filter = tokio::fs::read_to_string(&*FILTER_PATH).await?; let filter = tokio::fs::read_to_string(&*FILTER_PATH).await?;
db_mem.query(filter).await?; db_mem.query(filter).await?;
@ -189,6 +190,9 @@ pub async fn create_db_entities_bulk_filter(
db.import(format!("../data/temp{}.surql", file_name)) db.import(format!("../data/temp{}.surql", file_name))
.await?; .await?;
if let Some(ref p) = pb {
p.inc(batch_size as u64)
}
Ok(()) Ok(())
} }