NexVeridian/wikidata-to-surrealdb
1
0
Fork 0
mirror of https://github.com/NexVeridian/wikidata-to-surrealdb.git synced 2025-09-02 09:59:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: remove lazy_static and make tables.rs async

Browse source
This commit is contained in:
Elijah McMorris 2024-09-24 14:28:15 -07:00
parent dbd039eebf
commit 5fa50cf69f
Signed by: NexVeridian
SSH key fingerprint: SHA256:bsA1SKZxuEcEVHAy3gY1HUeM5ykRJl0U0kQHQn0hMg8
8 changed files with 142 additions and 83 deletions
Download patch file Download diff file Expand all files Collapse all files

66
src/utils/tables.rs
View file

@ -1,13 +1,16 @@
use lazy_static::lazy_static;
use futures::future::join_all;
use serde::{Deserialize, Serialize};
use std::env;
use surrealdb::sql::{Id, Thing};
use tokio::sync::OnceCell;
use wikidata::{ClaimValue, ClaimValueData, Entity, Lang, Pid, WikiId};
lazy_static! {
static ref WIKIDATA_LANG: String = env::var("WIKIDATA_LANG")
.expect("WIKIDATA_LANG not set")
.to_string();
static WIKIDATA_LANG: OnceCell<String> = OnceCell::const_new();
async fn get_wikidata_lang() -> &'static String {
WIKIDATA_LANG
.get_or_init(|| async { env::var("WIKIDATA_LANG").expect("WIKIDATA_LANG not set") })
.await
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@ -17,7 +20,7 @@ pub enum ClaimData {
}
impl ClaimData {
fn from_cvd(cvd: ClaimValueData) -> Self {
async fn from_cvd(cvd: ClaimValueData) -> Self {
match cvd {
ClaimValueData::Item(qid) => ClaimData::Thing(Thing::from(("Entity", Id::from(qid.0)))),
ClaimValueData::Property(pid) => {
@ -54,48 +57,53 @@ pub struct EntityMini {
}
impl EntityMini {
pub fn from_entity(entity: Entity) -> (Claims, Self) {
let thing_claim = Thing::from(("Claims", get_id_entity(&entity).id));
pub async fn from_entity(entity: Entity) -> (Claims, Self) {
let thing_claim = Thing::from(("Claims", get_id_entity(&entity).await.id));
(
Claims {
id: Some(thing_claim.clone()),
..Self::flatten_claims(entity.claims.clone())
..Self::flatten_claims(entity.claims.clone()).await
},
Self {
id: Some(get_id_entity(&entity)),
label: get_name(&entity),
id: Some(get_id_entity(&entity).await),
label: get_name(&entity).await,
claims: thing_claim,
description: get_description(&entity),
description: get_description(&entity).await,
},
)
}
fn flatten_claims(claims: Vec<(Pid, ClaimValue)>) -> Claims {
async fn flatten_claims(claims: Vec<(Pid, ClaimValue)>) -> Claims {
Claims {
id: None,
claims: claims
.iter()
.flat_map(|(pid, claim_value)| {
claims: {
let futures = claims.iter().map(|(pid, claim_value)| async {
let mut flattened = vec![Claim {
id: Thing::from(("Property", Id::from(pid.0))),
value: ClaimData::from_cvd(claim_value.data.clone()),
value: ClaimData::from_cvd(claim_value.data.clone()).await,
}];
flattened.extend(claim_value.qualifiers.iter().map(
|(qualifier_pid, qualifier_value)| Claim {
id: Thing::from(("Claims", Id::from(qualifier_pid.0))),
value: ClaimData::from_cvd(qualifier_value.clone()),
let inner_futures = claim_value.qualifiers.iter().map(
|(qualifier_pid, qualifier_value)| async {
let qualifier_data = ClaimData::from_cvd(qualifier_value.clone()).await;
Claim {
id: Thing::from(("Claims", Id::from(qualifier_pid.0))),
value: qualifier_data,
}
},
));
);
flattened.extend(join_all(inner_futures).await);
flattened
})
.collect(),
});
join_all(futures).await.into_iter().flatten().collect()
},
}
}
}
fn get_id_entity(entity: &Entity) -> Thing {
async fn get_id_entity(entity: &Entity) -> Thing {
let (id, tb) = match entity.id {
WikiId::EntityId(qid) => (qid.0, "Entity".to_string()),
WikiId::PropertyId(pid) => (pid.0, "Property".to_string()),
@ -106,18 +114,18 @@ fn get_id_entity(entity: &Entity) -> Thing {
Thing::from((tb, Id::from(id)))
}
fn get_name(entity: &Entity) -> String {
async fn get_name(entity: &Entity) -> String {
entity
.labels
.get(&Lang(WIKIDATA_LANG.to_string()))
.get(&Lang(get_wikidata_lang().await.to_string()))
.map(|label| label.to_string())
.unwrap_or_default()
}
fn get_description(entity: &Entity) -> String {
async fn get_description(entity: &Entity) -> String {
entity
.descriptions
.get(&Lang(WIKIDATA_LANG.to_string()))
.get(&Lang(get_wikidata_lang().await.to_string()))
.cloned()
.unwrap_or_default()
}