NexVeridian/wikidata-to-surrealdb
1
0
Fork 0
mirror of https://github.com/NexVeridian/wikidata-to-surrealdb.git synced 2025-09-02 01:49:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix!: remove OVERWRITE_DB, swap to querry

Browse source
This commit is contained in:
Elijah McMorris 2024-09-26 12:49:24 -07:00
parent d9803555d1
commit 8df3ab1d74
Signed by: NexVeridian
SSH key fingerprint: SHA256:bsA1SKZxuEcEVHAy3gY1HUeM5ykRJl0U0kQHQn0hMg8
5 changed files with 28 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

20
Cargo.lock generated
View file

@ -216,9 +216,9 @@ dependencies = [
[[package]]
name = "async-graphql"
version = "7.0.10"
version = "7.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a19415d9541f1758f39bdf0c732848beb7e2e39df9b32f90c6635882c3f9173a"
checksum = "0ba6d24703c5adc5ba9116901b92ee4e4c0643c01a56c4fd303f3818638d7449"
dependencies = [
"async-graphql-derive",
"async-graphql-parser",
@ -247,9 +247,9 @@ dependencies = [
[[package]]
name = "async-graphql-derive"
version = "7.0.10"
version = "7.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73a85254454f63ae1e5a475afff931465f11bf76d19fb5bb1b1d0d6a2f2b8db0"
checksum = "a94c2d176893486bd37cd1b6defadd999f7357bf5804e92f510c08bcf16c538f"
dependencies = [
"Inflector",
"async-graphql-parser",
@ -264,9 +264,9 @@ dependencies = [
[[package]]
name = "async-graphql-parser"
version = "7.0.10"
version = "7.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e94b202e404d18429c8482d61f64cb0a8639fd1e7c2caf2b258f035e0b7caff"
checksum = "79272bdbf26af97866e149f05b2b546edb5c00e51b5f916289931ed233e208ad"
dependencies = [
"async-graphql-value",
"pest",
@ -276,9 +276,9 @@ dependencies = [
[[package]]
name = "async-graphql-value"
version = "7.0.10"
version = "7.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a43a7bbb0ddea47c6f51913eba6e17a093b34e000588a93bb80a978ad129f3e9"
checksum = "ef5ec94176a12a8cbe985cd73f2e54dc9c702c88c766bdef12f1f3a67cedbee1"
dependencies = [
"bytes 1.7.2",
"indexmap 2.5.0",
@ -2752,9 +2752,9 @@ dependencies = [
[[package]]
name = "reblessive"
version = "0.4.0"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "568fde39e6aec674be99c9dd38b4c79040faf31038bd5a41ab1908db00c2319b"
checksum = "1d4f118ca848dfd632a8c0883f9aacd6b58da548eb0629a78cafee3d330938da"
[[package]]
name = "redox_syscall"

2
README.md
View file

@ -53,8 +53,6 @@ WIKIDATA_FILE_FORMAT=bz2
WIKIDATA_FILE_NAME=data/latest-all.json.bz2
# If not using docker file for Wikidata to SurrealDB, use 0.0.0.0:8000
WIKIDATA_DB_PORT=surrealdb:8000
# true=overwrite existing data, false=skip if already exists
OVERWRITE_DB=false
CREATE_VERSION=Bulk
#FILTER_PATH=data/filter.surql
```

18
flake.lock generated
View file

@ -3,11 +3,11 @@
"advisory-db": {
"flake": false,
"locked": {
"lastModified": 1727306865,
"narHash": "sha256-okbcHbpPc9wDVrppp8kQDOWd+PBwlHNkbiCLQ/GArvM=",
"lastModified": 1727353582,
"narHash": "sha256-2csMEEOZhvowVKZNBHk1kMJqk72ZMrPj9LQYCzP6EKs=",
"owner": "rustsec",
"repo": "advisory-db",
"rev": "6960f548f3abd498f948105b627d33065e23e37c",
"rev": "cb905e6e405834bdff1eb1e20c9b10edb5403889",
"type": "github"
},
"original": {
@ -39,11 +39,11 @@
"rust-analyzer-src": []
},
"locked": {
"lastModified": 1727245890,
"narHash": "sha256-B4gUhZxqdn24PqL7z7ZuvLOS84HVskhKRByWdgA4/RI=",
"lastModified": 1727332394,
"narHash": "sha256-dBYQD4DPxu/hBndSbfMA5HhHrVnrxrW9Ju8R3augGzw=",
"owner": "nix-community",
"repo": "fenix",
"rev": "de3acda8b67b92abeeb35ac236924afd959874ad",
"rev": "60a35a47e8ae3721efaae2229ec6037e3fde2d17",
"type": "github"
},
"original": {
@ -72,11 +72,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1727173215,
"narHash": "sha256-OtMlWYCqBDbnEsByoows785Gem9CSMiXYEBiKKtStk4=",
"lastModified": 1727296349,
"narHash": "sha256-C3SRU3GMDNII9l16o4+nkybuxaDX4x5TBypwmmUBCo0=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "965289e5e07243f1cde3212d8bcaf726d36c5c46",
"rev": "fe866c653c24adf1520628236d4e70bbb2fdd949",
"type": "github"
},
"original": {

46
src/utils.rs
View file

@ -17,20 +17,8 @@ pub mod init_reader;
mod tables;
use tables::*;
static OVERWRITE_DB: OnceCell<bool> = OnceCell::const_new();
static FILTER_PATH: OnceCell<String> = OnceCell::const_new();
async fn get_overwrite_db() -> bool {
*OVERWRITE_DB
.get_or_init(|| async {
env::var("OVERWRITE_DB")
.expect("OVERWRITE_DB not set")
.parse::<bool>()
.expect("Failed to parse OVERWRITE_DB")
})
.await
}
async fn get_filter_path() -> &'static String {
FILTER_PATH
.get_or_init(|| async {
@ -160,33 +148,15 @@ impl CreateVersion {
claims_vec.push(claims);
}
if get_overwrite_db().await {
db.upsert::<Vec<EntityMini>>("Entity")
.content(entity_vec)
db.query("INSERT INTO Entity $entity_vec RETURN NONE;")
.bind(("entity_vec", entity_vec))
.query("INSERT INTO Claims $claims_vec RETURN NONE;")
.bind(("claims_vec", claims_vec))
.query("INSERT INTO Property $property_vec RETURN NONE;")
.bind(("property_vec", property_vec))
.query("INSERT INTO Lexeme $lexeme_vec RETURN NONE;")
.bind(("lexeme_vec", lexeme_vec))
.await?;
db.upsert::<Vec<Claims>>("Claims")
.content(claims_vec)
.await?;
db.upsert::<Vec<EntityMini>>("Property")
.content(property_vec)
.await?;
db.upsert::<Vec<EntityMini>>("Lexeme")
.content(lexeme_vec)
.await?;
} else {
db.insert::<Vec<EntityMini>>("Entity")
.content(entity_vec)
.await?;
db.insert::<Vec<Claims>>("Claims")
.content(claims_vec)
.await?;
db.insert::<Vec<EntityMini>>("Property")
.content(property_vec)
.await?;
db.insert::<Vec<EntityMini>>("Lexeme")
.content(lexeme_vec)
.await?;
}
if let Some(ref p) = pb {
p.inc(batch_size as u64)

1
tests/integration.rs
View file

@ -8,7 +8,6 @@ use wikidata_to_surrealdb::utils::*;
async fn inti_db() -> Result<Surreal<Db>, Error> {
env::set_var("WIKIDATA_LANG", "en");
env::set_var("OVERWRITE_DB", "false");
let db = init_db::create_db_mem().await?;