From 4663a862a4a0b28324bc26dcfcfc873b50fbfc5c Mon Sep 17 00:00:00 2001 From: NexVeridian Date: Wed, 18 Sep 2024 20:05:37 -0700 Subject: [PATCH] feat!: surrealdb 2.0, remove CreateVersion::Single, fix OVERWRITE_DB --- CONTRIBUTING.md | 4 +- Cargo.lock | 383 +++++++++++++++++++++---------- Cargo.toml | 4 +- Makefile | 7 + README.md | 1 - Useful queries.md | 17 ++ benches/bench.rs | 17 -- docker-compose-surrealdb.dev.yml | 2 +- docker-compose-surrealdb.yml | 2 +- flake.lock | 18 +- src/main.rs | 1 - src/utils.rs | 89 +++---- tests/integration.rs | 28 +-- 13 files changed, 328 insertions(+), 245 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8944096..ebdda09 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,5 @@ # Contributing code -- Make sure the test pass `cargo t` -- Run `cargo clippy --fix --allow-dirty` -- Run `cargo bench` +- Run `make precommit` # Dev Install ## Dev Containers diff --git a/Cargo.lock b/Cargo.lock index 231d871..d25ac3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,16 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "addr" version = "0.15.6" @@ -160,10 +170,16 @@ dependencies = [ ] [[package]] -name = "arrayvec" -version = "0.7.6" +name = "arrayref" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "ascii-canvas" @@ -199,10 +215,93 @@ dependencies = [ ] [[package]] -name = "async-recursion" -version = "1.1.1" +name = "async-graphql" +version = "7.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +checksum = "9d37c3e9ba322eb00e9e5e997d58f08e8b6de037325b9367ac59bca8e3cd46af" +dependencies = [ + "async-graphql-derive", + "async-graphql-parser", + "async-graphql-value", + "async-stream", + "async-trait", + "base64 0.22.1", + "bytes 1.7.2", + "fnv", + "futures-timer", + "futures-util", + "http", + "indexmap 2.5.0", + "mime", + "multer", + "num-traits", + "once_cell", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "static_assertions_next", + "thiserror", +] + +[[package]] +name = "async-graphql-derive" +version = "7.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1141703c11c6ad4fa9b3b0e1e476dea01dbd18a44db00f949b804afaab2f344" +dependencies = [ + "Inflector", + "async-graphql-parser", + "darling", + "proc-macro-crate", + "proc-macro2", + "quote", + "strum", + "syn 2.0.77", + "thiserror", +] + +[[package]] +name = "async-graphql-parser" +version = "7.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f66edcce4c38c18f7eb181fdf561c3d3aa2d644ce7358fc7a928c00a4ffef17" +dependencies = [ + "async-graphql-value", + "pest", + "serde", + "serde_json", +] + +[[package]] +name = "async-graphql-value" +version = "7.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0206011cad065420c27988f17dd7fe201a0e056b20c262209b7bffcd6fa176" +dependencies = [ + "bytes 1.7.2", + "indexmap 2.5.0", + "serde", + "serde_json", +] + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", @@ -237,15 +336,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "atomic-polyfill" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" -dependencies = [ - "critical-section", -] - [[package]] name = "autocfg" version = "1.3.0" @@ -361,6 +451,19 @@ dependencies = [ "digest", ] +[[package]] +name = "blake3" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -456,9 +559,12 @@ dependencies = [ [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +dependencies = [ + "serde", +] [[package]] name = "bzip2" @@ -500,9 +606,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.19" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d74707dde2ba56f86ae90effb3b43ddd369504387e718014de010cec7959800" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "shlex", ] @@ -675,6 +781,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -737,12 +849,6 @@ dependencies = [ "itertools 0.10.5", ] -[[package]] -name = "critical-section" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64009896348fc5af4222e9cf7d7d82a95a256c634ebcf61c53e4ea461422242" - [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -953,6 +1059,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1196,26 +1311,9 @@ dependencies = [ [[package]] name = "geo" -version = "0.26.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1645cf1d7fea7dac1a66f7357f3df2677ada708b8d9db8e9b043878930095a96" -dependencies = [ - "earcutr", - "float_next_after", - "geo-types", - "geographiclib-rs", - "log", - "num-traits", - "robust", - "rstar", - "serde", -] - -[[package]] -name = "geo" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4841b40fdbccd4b7042bd6195e4de91da54af34c50632e371bcbfcdfb558b873" +checksum = "f811f663912a69249fa620dcd2a005db7254529da2d8a0b23942e81f47084501" dependencies = [ "earcutr", "float_next_after", @@ -1288,9 +1386,9 @@ dependencies = [ [[package]] name = "hash32" -version = "0.2.1" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" dependencies = [ "byteorder", ] @@ -1316,14 +1414,11 @@ dependencies = [ [[package]] name = "heapless" -version = "0.7.17" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" dependencies = [ - "atomic-polyfill", "hash32", - "rustc_version", - "spin", "stable_deref_trait", ] @@ -1333,6 +1428,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -1380,7 +1481,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "fnv", "itoa", ] @@ -1391,7 +1492,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "http", ] @@ -1401,7 +1502,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-util", "http", "http-body", @@ -1426,7 +1527,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-channel", "futures-util", "http", @@ -1463,7 +1564,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-channel", "futures-util", "http", @@ -1876,12 +1977,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -1903,6 +1998,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes 1.7.2", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "nanoid" version = "0.4.0" @@ -1976,16 +2088,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -2062,7 +2164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" dependencies = [ "async-trait", - "bytes 1.7.1", + "bytes 1.7.2", "chrono", "futures 0.3.30", "humantime", @@ -2168,6 +2270,17 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c73c26c01b8c87956cea613c907c9d6ecffd8d18a2a5908e5de0adfaa185cea" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + [[package]] name = "petgraph" version = "0.6.5" @@ -2401,7 +2514,6 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn 1.0.109", "version_check", ] @@ -2503,7 +2615,7 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "pin-project-lite", "quinn-proto", "quinn-udp", @@ -2521,7 +2633,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "rand", "ring", "rustc-hash", @@ -2733,7 +2845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "base64 0.22.1", - "bytes 1.7.1", + "bytes 1.7.2", "futures-core", "futures-util", "http", @@ -2773,30 +2885,25 @@ dependencies = [ [[package]] name = "revision" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98b99dba8f2787c9af2e46b17ff38437d213d46c8970b550e6b79b862bf7629" +checksum = "22f53179a035f881adad8c4d58a2c599c6b4a8325b989c68d178d7a34d1b1e4c" dependencies = [ - "bincode", "chrono", - "geo 0.26.0", + "geo", "regex", "revision-derive", "roaring", "rust_decimal", - "serde", - "thiserror", "uuid", ] [[package]] name = "revision-derive" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3721b4a8e52f9e52c54f74f482a4f550601f5c44cb7876606a2ab79cb09469c1" +checksum = "5f0ec466e5d8dca9965eb6871879677bef5590cf7525ad96cae14376efb75073" dependencies = [ - "darling", - "proc-macro-error", "proc-macro2", "quote", "syn 2.0.77", @@ -2825,7 +2932,7 @@ checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec", "bytecheck", - "bytes 1.7.1", + "bytes 1.7.2", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -2897,9 +3004,9 @@ checksum = "cbf4a6aa5f6d6888f39e980649f3ad6b666acdce1d78e95b8a2cb076e687ae30" [[package]] name = "rstar" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73111312eb7a2287d229f06c00ff35b51ddee180f017ab6dec1f69d62ac098d6" +checksum = "133315eb94c7b1e8d0cb097e5a710d850263372fd028fff18969de708afc7008" dependencies = [ "heapless", "num-traits", @@ -2954,7 +3061,7 @@ checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" dependencies = [ "arrayvec", "borsh", - "bytes 1.7.1", + "bytes 1.7.2", "num-traits", "rand", "rkyv", @@ -3295,7 +3402,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 1.0.109", @@ -3334,9 +3441,6 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] [[package]] name = "stable_deref_trait" @@ -3357,6 +3461,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "static_assertions_next" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7beae5182595e9a8b683fa98c4317f956c9a2dec3b9716990d20023cc60c766" + [[package]] name = "storekey" version = "0.5.0" @@ -3401,6 +3511,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.77", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3408,19 +3540,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] -name = "surrealdb-alpha" -version = "2.0.9" +name = "surrealdb" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff118b6c96463359da620c4306a78fcd65ab07cf4b1dda324bb81c3e71d8c5d5" +checksum = "54a327b352feacd0e724399dcb6a673acd6fb2a88e0dc9c3fed7758d97dceef1" dependencies = [ + "arrayvec", "async-channel", "bincode", "chrono", "dmp", "futures 0.3.30", - "geo 0.27.0", + "geo", "indexmap 2.5.0", - "once_cell", "path-clean", "pharos", "reblessive", @@ -3434,7 +3566,7 @@ dependencies = [ "serde", "serde-content", "serde_json", - "surrealdb-core-alpha", + "surrealdb-core", "thiserror", "tokio", "tokio-tungstenite", @@ -3449,10 +3581,10 @@ dependencies = [ ] [[package]] -name = "surrealdb-core-alpha" -version = "2.0.9" +name = "surrealdb-core" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e7ce9dcb41fa585fdf02b59d7a1cc169acb2e82f4e7000974291e997e548b4" +checksum = "80433611bdf00df5ff59d52a708b60ac23598db3e3f9460cde625ea925c8f776" dependencies = [ "addr", "ahash 0.8.11", @@ -3461,11 +3593,12 @@ dependencies = [ "argon2", "async-channel", "async-executor", - "async-recursion", + "async-graphql", "base64 0.21.7", "bcrypt", "bincode", - "bytes 1.7.1", + "blake3", + "bytes 1.7.2", "castaway", "cedar-policy", "chrono", @@ -3478,7 +3611,7 @@ dependencies = [ "fst", "futures 0.3.30", "fuzzy-matcher", - "geo 0.27.0", + "geo", "geo-types", "hex", "ipnet", @@ -3489,11 +3622,9 @@ dependencies = [ "nanoid", "ndarray", "ndarray-stats", - "nom", "num-traits", "num_cpus", "object_store", - "once_cell", "pbkdf2", "pharos", "phf", @@ -3546,9 +3677,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "12.11.0" +version = "12.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1db5ac243c7d7f8439eb3b8f0357888b37cf3732957e91383b0ad61756374e" +checksum = "9fdf97c441f18a4f92425b896a4ec7a27e03631a0b1047ec4e34e9916a9a167e" dependencies = [ "debugid", "memmap2", @@ -3558,9 +3689,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.11.0" +version = "12.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea26e430c27d4a8a5dea4c4b81440606c7c1a415bd611451ef6af8c81416afc3" +checksum = "bc8ece6b129e97e53d1fbb3f61d33a6a9e5369b11d01228c068094d6d134eaea" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -3753,7 +3884,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", - "bytes 1.7.1", + "bytes 1.7.2", "libc", "mio", "pin-project-lite", @@ -3817,7 +3948,7 @@ version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-core", "futures-io", "futures-sink", @@ -3833,9 +3964,9 @@ checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" dependencies = [ "indexmap 2.5.0", "toml_datetime", @@ -3924,7 +4055,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" dependencies = [ "byteorder", - "bytes 1.7.1", + "bytes 1.7.2", "data-encoding", "http", "httparse", @@ -3944,6 +4075,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "ulid" version = "1.1.3" @@ -3979,9 +4116,9 @@ checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -4200,9 +4337,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] @@ -4234,7 +4371,7 @@ dependencies = [ "rstest", "serde", "serde_json", - "surrealdb-alpha", + "surrealdb", "tokio", "wikidata", ] diff --git a/Cargo.toml b/Cargo.toml index 5aa4eb7..b020128 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT OR Apache-2.0" anyhow = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -surrealdb-alpha = { version = "2.0.9", features = ["protocol-http", "kv-mem"] } +surrealdb = { version = "2.0.1", features = ["protocol-http", "kv-mem"] } tokio = { version = "1.39", features = ["fs", "time"] } futures = "0.3" wikidata = "1.1" @@ -19,7 +19,7 @@ rand = "0.8" [dev-dependencies] rstest = "0.22" -surrealdb-alpha = { version = "2.0.9", features = ["kv-mem"] } +surrealdb = { version = "2.0.1", features = ["kv-mem"] } criterion = { version = "0.5", features = ["async_tokio"] } pprof = { version = "0.13", features = ["criterion", "protobuf-codec"] } diff --git a/Makefile b/Makefile index 6d4c233..0258fd0 100644 --- a/Makefile +++ b/Makefile @@ -12,3 +12,10 @@ $(addprefix down-,$(COMPOSE_FILES)): view: docker attach wikidata-to-surrealdb + +precommit: + rustup update + cargo update + cargo fmt + cargo t + cargo clippy --fix --allow-dirty diff --git a/README.md b/README.md index d7314d0..8d208cb 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,6 @@ CREATE_VERSION=Bulk Env string CREATE_VERSION must be in the enum CREATE_VERSION ```rust pub enum CreateVersion { - Single, #[default] Bulk, /// must create a filter.surql file in the data directory diff --git a/Useful queries.md b/Useful queries.md index 5670b76..68b1a37 100644 --- a/Useful queries.md +++ b/Useful queries.md @@ -11,6 +11,23 @@ delete $claims; delete $entity; ``` +# Create a view for media +``` +DEFINE TABLE Media TYPE NORMAL AS +SELECT +*, +# Number of episodes +(claims.claims[WHERE id = Property:1113].value.ClaimValueData.Quantity.amount)[0] AS episodes, +# Part of the series (parent) +(claims.claims[WHERE id = Property:179].value.Thing)[0] AS parent, +# Has part(s) (children) +claims.claims[WHERE id = Property:527].value.Thing AS children +FROM Entity; + +# Full-Text Search +DEFINE ANALYZER OVERWRITE blank_english TOKENIZERS blank FILTERS lowercase, ascii, snowball(english); +``` + # Select for media ``` SELECT diff --git a/benches/bench.rs b/benches/bench.rs index c95f4cd..4c331c8 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -20,23 +20,6 @@ async fn inti_db() -> Result, Error> { fn bench(c: &mut Criterion) { let mut group = c.benchmark_group("Create DB Entities"); - group.bench_function("Single Insert", |b| { - b.iter(|| { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - let db = inti_db().await.unwrap(); - let reader = File_Format::new("json") - .reader("tests/data/bench.json") - .unwrap(); - - CreateVersion::Single - .run(Some(db.clone()), reader, None, 1000, 100) - .await - .unwrap(); - }) - }) - }); - group.bench_function("Bulk Insert", |b| { b.iter(|| { let rt = Runtime::new().unwrap(); diff --git a/docker-compose-surrealdb.dev.yml b/docker-compose-surrealdb.dev.yml index 0e7ae7b..9be4763 100644 --- a/docker-compose-surrealdb.dev.yml +++ b/docker-compose-surrealdb.dev.yml @@ -1,7 +1,7 @@ services: surrealdb: container_name: surrealdb - image: surrealdb/surrealdb:v2.0.0-alpha.9 + image: surrealdb/surrealdb:v2.0.1 env_file: - .env entrypoint: diff --git a/docker-compose-surrealdb.yml b/docker-compose-surrealdb.yml index 93f6e7e..5d1ffbf 100644 --- a/docker-compose-surrealdb.yml +++ b/docker-compose-surrealdb.yml @@ -1,7 +1,7 @@ services: surrealdb: container_name: surrealdb - image: surrealdb/surrealdb:v2.0.0-alpha.9 + image: surrealdb/surrealdb:v2.0.1 env_file: - .env entrypoint: diff --git a/flake.lock b/flake.lock index 0287f81..29b6fdb 100644 --- a/flake.lock +++ b/flake.lock @@ -39,11 +39,11 @@ "rust-analyzer-src": [] }, "locked": { - "lastModified": 1726468443, - "narHash": "sha256-O1VcbVBrqIf58U05yFXl9+J7XM2qh0I+7vqMbNwZPq0=", + "lastModified": 1726641202, + "narHash": "sha256-NrSmOWnr0bIudOLwXd7UwspaHCGwVp8F0jed+nleWpI=", "owner": "nix-community", "repo": "fenix", - "rev": "effac20e9560aab202e82b6d833f685163a9c138", + "rev": "59e6cc52c6242800bb448c5a2b6427bd949385ad", "type": "github" }, "original": { @@ -57,11 +57,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", "type": "github" }, "original": { @@ -72,11 +72,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1726481836, - "narHash": "sha256-MWTBH4dd5zIz2iatDb8IkqSjIeFum9jAqkFxgHLdzO4=", + "lastModified": 1726583932, + "narHash": "sha256-zACxiQx8knB3F8+Ze+1BpiYrI+CbhxyWpcSID9kVhkQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "20f9370d5f588fb8c72e844c54511cab054b5f40", + "rev": "658e7223191d2598641d50ee4e898126768fe847", "type": "github" }, "original": { diff --git a/src/main.rs b/src/main.rs index eab7f37..39905ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,6 @@ lazy_static! { .expect("CREATE_VERSION not set") .as_str() { - "Single" => CreateVersion::Single, "Bulk" => CreateVersion::Bulk, "BulkFilter" => CreateVersion::BulkFilter, _ => panic!("Unknown CREATE_VERSION"), diff --git a/src/utils.rs b/src/utils.rs index 6b47877..fae4fd8 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -25,34 +25,8 @@ lazy_static! { env::var("FILTER_PATH").unwrap_or("data/filter.surql".to_string()); } -pub async fn create_entity(db: &Surreal, line: &str) -> Result<(), Error> { - let line = line.trim().trim_end_matches(',').to_string(); - if line == "[" || line == "]" { - return Ok(()); - } - - let json: Value = from_str(&line)?; - let data = Entity::from_json(json).expect("Failed to parse JSON"); - - let (mut claims, mut data) = EntityMini::from_entity(data); - - let id = data.id.clone().expect("No ID"); - data.id = None; - if db.create::>(&id).await.is_err() && *OVERWRITE_DB { - db.update::>(&id).content(data).await?; - } - - let id = claims.id.clone().expect("No ID"); - claims.id = None; - if db.create::>(&id).await.is_err() && *OVERWRITE_DB { - db.update::>(&id).content(claims).await?; - } - Ok(()) -} - #[derive(Clone, Copy, Default)] pub enum CreateVersion { - Single, #[default] Bulk, /// must create a filter.surql file in the root directory @@ -136,7 +110,6 @@ impl CreateVersion { batch_size: usize, ) -> bool { match self { - CreateVersion::Single => self.create_single(db, chunk, pb).await.is_ok(), CreateVersion::Bulk => self.create_bulk(db, chunk, pb, batch_size).await.is_ok(), CreateVersion::BulkFilter => self .create_bulk_filter(db, chunk, pb, batch_size) @@ -151,25 +124,6 @@ impl CreateVersion { } } - async fn create_single( - self, - db: &Surreal, - lines: &[String], - pb: &Option, - ) -> Result<(), Error> { - let mut counter = 0; - for line in lines { - create_entity(db, line).await?; - counter += 1; - if counter % 100 == 0 { - if let Some(ref p) = pb { - p.inc(100) - } - } - } - Ok(()) - } - async fn create_bulk( self, db: &Surreal, @@ -183,7 +137,7 @@ impl CreateVersion { .filter(|line| line != "[" && line != "]") .collect::>(); - let mut data_vec: Vec = Vec::with_capacity(batch_size); + let mut entity_vec: Vec = Vec::with_capacity(batch_size); let mut claims_vec: Vec = Vec::with_capacity(batch_size); let mut property_vec: Vec = Vec::with_capacity(batch_size); let mut lexeme_vec: Vec = Vec::with_capacity(batch_size); @@ -195,24 +149,39 @@ impl CreateVersion { match data.id.clone().expect("No ID").tb.as_str() { "Property" => property_vec.push(data), "Lexeme" => lexeme_vec.push(data), - "Entity" => data_vec.push(data), + "Entity" => entity_vec.push(data), _ => panic!("Unknown table"), } claims_vec.push(claims); } - db.insert::>("Entity") - .content(data_vec) - .await?; - db.insert::>("Claims") - .content(claims_vec) - .await?; - db.insert::>("Property") - .content(property_vec) - .await?; - db.insert::>("Lexeme") - .content(lexeme_vec) - .await?; + if *OVERWRITE_DB { + db.upsert::>("Entity") + .content(entity_vec) + .await?; + db.upsert::>("Claims") + .content(claims_vec) + .await?; + db.upsert::>("Property") + .content(property_vec) + .await?; + db.upsert::>("Lexeme") + .content(lexeme_vec) + .await?; + } else { + db.insert::>("Entity") + .content(entity_vec) + .await?; + db.insert::>("Claims") + .content(claims_vec) + .await?; + db.insert::>("Property") + .content(property_vec) + .await?; + db.insert::>("Lexeme") + .content(lexeme_vec) + .await?; + } if let Some(ref p) = pb { p.inc(batch_size as u64) diff --git a/tests/integration.rs b/tests/integration.rs index 97b8240..a8b87b7 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -8,7 +8,7 @@ use wikidata_to_surrealdb::utils::*; async fn inti_db() -> Result, Error> { env::set_var("WIKIDATA_LANG", "en"); - env::set_var("OVERWRITE_DB", "true"); + env::set_var("OVERWRITE_DB", "false"); let db = init_db::create_db_mem().await?; @@ -37,20 +37,7 @@ async fn entity_query(db: &Surreal) -> Result, Error> { Ok(x) } -#[tokio::test] -async fn entity() { - let db = inti_db().await.unwrap(); - let reader = init_reader("json", "Entity"); - - for line in reader.lines() { - create_entity(&db, &line.unwrap()).await.unwrap(); - } - - assert_eq!(51.0, entity_query(&db).await.unwrap().unwrap()) -} - #[rstest] -#[case(CreateVersion::Single)] #[case(CreateVersion::Bulk)] #[tokio::test] async fn entity_threaded(#[case] version: CreateVersion) -> Result<(), Error> { @@ -96,20 +83,7 @@ async fn property_query(db: &Surreal) -> Result, Error> { Ok(x) } -#[tokio::test] -async fn property() { - let db = inti_db().await.unwrap(); - let reader = init_reader("json", "Property"); - - for line in reader.lines() { - create_entity(&db, &line.unwrap()).await.unwrap(); - } - - assert_eq!(2.0, property_query(&db).await.unwrap().unwrap()) -} - #[rstest] -#[case(CreateVersion::Single)] #[case(CreateVersion::Bulk)] #[tokio::test] async fn property_threaded(#[case] version: CreateVersion) -> Result<(), Error> {