diff --git a/.gitignore b/.gitignore index baaa686..99de4d0 100644 --- a/.gitignore +++ b/.gitignore @@ -103,7 +103,6 @@ venv.bak/ # mypy .mypy_cache/ +/data /target Cargo.lock - -data/* diff --git a/Cargo.toml b/Cargo.toml index c8c9517..205ef7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,10 @@ license = "MIT OR Apache-2.0" [dependencies] anyhow = "1.0" -dotenv_codegen = "0.15.0" +dotenv_codegen = "0.15" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" surrealdb = "1.0" tokio = "1.35" -wikidata = "0.3" +wikidata = "0.3.1" +bzip2 = { version = "0.4", features = ["tokio"] } diff --git a/docker-compose.yml b/docker-compose.yml index 09ba1b8..add4ee3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ version: "3" services: surrealdb: + image: surrealdb/surrealdb:latest container_name: surrealdb env_file: - .env @@ -12,7 +13,6 @@ services: - --pass - $DB_PASSWORD - file:/data/surrealdb - image: surrealdb/surrealdb:latest ports: - 8000:8000 volumes: diff --git a/src/main.rs b/src/main.rs index 483ac15..0806fdb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use anyhow::{Error, Ok, Result}; +use bzip2::read::MultiBzDecoder; use dotenv_codegen::dotenv; use serde_json::{from_str, Value}; use std::fs::File; @@ -9,20 +10,39 @@ use wikidata::Entity; mod utils; use utils::*; +#[allow(non_camel_case_types)] +enum File_Format { + json, + bz2, +} +impl File_Format { + fn new(file: &str) -> Self { + match file { + "json" => Self::json, + "bz2" => Self::bz2, + _ => panic!("Unknown file format"), + } + } + fn reader(self, file: &str) -> Result, Error> { + let file = File::open(file)?; + match self { + File_Format::json => Ok(Box::new(BufReader::new(file))), + File_Format::bz2 => Ok(Box::new(BufReader::new(MultiBzDecoder::new(file)))), + } + } +} + #[tokio::main] async fn main() -> Result<(), Error> { let db = Surreal::new::("0.0.0.0:8000").await?; - db.signin(Root { username: dotenv!("DB_USER"), password: dotenv!("DB_PASSWORD"), }) .await?; - db.use_ns("wikidata").use_db("wikidata").await?; - let file = File::open("data/e.json")?; - let reader = BufReader::new(file); + let reader = File_Format::new(dotenv!("FILE_FORMAT")).reader(dotenv!("FILE_NAME"))?; for line in reader.lines() { let line = line?.trim().trim_end_matches(',').to_string();