swap to crane

.cargo/config.toml

@@ -1,6 +1,6 @@
-[target.x86_64-unknown-linux-gnu]
+# [target.x86_64-unknown-linux-gnu]
-linker = "clang"
+# linker = "clang"
-rustflags = ["-C", "link-arg=-fuse-ld=/usr/bin/mold"]
+# rustflags = ["-C", "link-arg=-fuse-ld=/usr/bin/mold"]
 
 [alias]
 t = "nextest run"

.devcontainer/Dockerfile

@@ -4,16 +4,16 @@ FROM mcr.microsoft.com/devcontainers/rust:bookworm
 RUN apt-get update && \
 	apt install -y build-essential xz-utils musl-tools musl-dev gcc-multilib pkg-config libssl-dev clang mold
 
-RUN LAZYGIT_VERSION=$(curl -s "https://api.github.com/repos/jesseduffield/lazygit/releases/latest" | grep -Po '"tag_name": "v\K[0-9.]+') && \
+# RUN LAZYGIT_VERSION=$(curl -s "https://api.github.com/repos/jesseduffield/lazygit/releases/latest" | grep -Po '"tag_name": "v\K[0-9.]+') && \
-	curl -Lo lazygit.tar.gz "https://github.com/jesseduffield/lazygit/releases/latest/download/lazygit_${LAZYGIT_VERSION}_Linux_x86_64.tar.gz" && \
+# 	curl -Lo lazygit.tar.gz "https://github.com/jesseduffield/lazygit/releases/latest/download/lazygit_${LAZYGIT_VERSION}_Linux_x86_64.tar.gz" && \
-	sudo tar xf lazygit.tar.gz -C /usr/local/bin lazygit && \ 
+# 	sudo tar xf lazygit.tar.gz -C /usr/local/bin lazygit && \ 
-	rm -rf lazygit.tar.gz
+# 	rm -rf lazygit.tar.gz
 
-RUN BTOP_VERSION=$(curl -s "https://api.github.com/repos/aristocratos/btop/releases/latest" | grep -Po '"tag_name": "v\K[0-9.]+') && \ 
+# RUN BTOP_VERSION=$(curl -s "https://api.github.com/repos/aristocratos/btop/releases/latest" | grep -Po '"tag_name": "v\K[0-9.]+') && \ 
-	wget "https://github.com/aristocratos/btop/releases/download/v${BTOP_VERSION}/btop-x86_64-linux-musl.tbz" && \
+# 	wget "https://github.com/aristocratos/btop/releases/download/v${BTOP_VERSION}/btop-x86_64-linux-musl.tbz" && \
-	sudo tar -xvf btop-x86_64-linux-musl.tbz && \
+# 	sudo tar -xvf btop-x86_64-linux-musl.tbz && \
-	cd btop && ./install.sh && cd .. && \
+# 	cd btop && ./install.sh && cd .. && \
-	rm -rf btop-x86_64-linux-musl.tbz btop
+# 	rm -rf btop-x86_64-linux-musl.tbz btop
 
-RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+# RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
 # cargo install cargo-nextest --locked

.devcontainer/devcontainer.json

@@ -27,6 +27,11 @@
 			"source": "${localWorkspaceFolderBasename}-target",
 			"target": "${containerWorkspaceFolder}/target",
 			"type": "volume"
+		},
+		{
+			"source": "nix-store-${containerWorkspaceFolderBasename}",
+			"target": "/nix/store",
+			"type": "volume"
 		}
 	],
 	// Features to add to the dev container. More info: https://containers.dev/features.
@@ -51,7 +56,8 @@
 	"postAttachCommand": {
 		"AddGitSafeDir": "git config --global --add safe.directory /workspaces/${containerWorkspaceFolderBasename}",
 		"update": "rustup target add x86_64-unknown-linux-musl && rustup update && cargo update",
-		"clippy": "cargo clippy --fix --allow-dirty"
+		"clippy": "cargo clippy --fix --allow-dirty",
+		"nix flake update": "nix flake update"
 	},
 	// Configure tool-specific properties.
 	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
@@ -69,7 +75,8 @@
 				"ms-azuretools.vscode-docker",
 				"redhat.vscode-yaml",
 				"GitHub.copilot",
-				"GitHub.copilot-chat"
+				"GitHub.copilot-chat",
+				"jnoortheen.nix-ide"
 			]
 		}
 	}

.dockerignore

@@ -7,7 +7,9 @@
 .env
 .gitignore
 Cargo.lock
-docker-compose.dev.yml
+docker-compose-surrealdb.dev.yml
-docker-compose.yml
+docker-compose-surrealdb.yml
 dockerfile
 *.md
+Makefile
+flake.nix

.github/workflows/crane.yml

@@ -0,0 +1,28 @@
+# https://github.com/nextest-rs/reuse-build-partition-example
+# https://keliris.dev/articles/setup-rust-github-actions
+
+name: crane
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  schedule:
+    - cron: 0 0 * * 1
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  check:
+    name: check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix flake check --all-systems

.github/workflows/docker.yml

@@ -1,19 +1,15 @@
-name: Docker
+name: docker
 
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
 on:
   workflow_run:
-    workflows: [nextest]
+    workflows: [crane]
     branches: [main]
     types:
       - completed
   # schedule:
   #   - cron: 0 0 * * 1
   # push:
-  #   branches: [ "main" ]
+  #   branches: ["main"]
   #   # Publish semver tags as releases.
   #   tags: [ 'v*.*.*' ]
   # pull_request:
@@ -41,25 +37,22 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      # Install the cosign tool except on PR
+      # Install Nix
-      # https://github.com/sigstore/cosign-installer
+      - name: Install Nix
-      - name: Install cosign
+        uses: DeterminateSystems/nix-installer-action@main
-        if: github.event_name != 'pull_request'
+      - uses: DeterminateSystems/magic-nix-cache-action@main
-        uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1
-        with:
-          cosign-release: "v2.1.1"
 
       # Set up BuildKit Docker container builder to be able to build
       # multi-platform images and export cache
       # https://github.com/docker/setup-buildx-action
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+        uses: docker/setup-buildx-action@v3
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -69,41 +62,27 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
+      # Build Nix package
+      - name: Build Nix package
+        run: nix build .#dockerImage
+
       # https://github.com/orgs/community/discussions/25768#discussioncomment-3249183
-      - name: downcase REPO
+      - name: Downcase REPO
         run: |
-          echo "REPO=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV}
+          echo "REPO=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV}
 
-      # Build and push Docker image with Buildx (don't push on PR)
+      - name: Strip REPO Username
-      # https://github.com/docker/build-push-action
+        run: |
-      - name: Build and push Docker image
+          STRIP_REPO_USERNAME=$(echo "${{ env.REPO }}" | sed 's/nexveridian\///')
-        id: build-and-push
+          echo "STRIP_REPO_USERNAME=${STRIP_REPO_USERNAME}" >> ${GITHUB_ENV}
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
-        with:
-          context: .
-          push: ${{ github.event_name != 'pull_request' }}
-          # tags: ${{ steps.meta.outputs.tags }}
-          # tags: ${{ env.REGISTRY }}/${{ github.repository }}:latest
-          tags: ${{ env.REGISTRY }}/${{ env.REPO }}:latest
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
 
-      # # Sign the resulting Docker image digest except on PRs.
+      # https://github.com/docker/build-push-action/issues/538
-      # # This will only write to the public Rekor transparency log when the Docker
+      - name: Push and tag Docker image
-      # # repository is public to avoid leaking data.  If you would like to publish
+        run: |
-      # # transparency data even for private images, pass --force to cosign below.
+          docker load < result
-      # # https://github.com/sigstore/cosign
+          docker tag ${{ env.STRIP_REPO_USERNAME }}:latest ${{ env.REGISTRY }}/${{ env.REPO }}:latest
-      # - name: Sign the published Docker image
+          docker push ${{ env.REGISTRY }}/${{ env.REPO }}:latest
-      #   if: ${{ github.event_name != 'pull_request' }}
-      #   env:
-      #     # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
-      #     TAGS: ${{ steps.meta.outputs.tags }}
-      #     DIGEST: ${{ steps.build-and-push.outputs.digest }}
-      #   # This step uses the identity token to provision an ephemeral certificate
-      #   # against the sigstore community Fulcio instance.
-      #   run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

.github/workflows/nextest.yml

@@ -1,45 +0,0 @@
-# https://github.com/nextest-rs/reuse-build-partition-example
-# https://keliris.dev/articles/setup-rust-github-actions
-
-name: nextest
-
-on:
-  pull_request:
-    branches: [main]
-  push:
-    branches: [main]
-  schedule:
-    - cron: 0 0 * * 1
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  run-tests:
-    name: run tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      # - uses: rui314/setup-mold@v1
-      - name: install mold
-        run: sudo apt-get install -y musl-tools musl-dev libssl-dev clang mold
-      # https://github.com/moonrepo/setup-rust
-      - uses: moonrepo/setup-rust@v1
-        with:
-          bins: cargo-nextest
-      - name: Run tests
-        run: cargo nextest run -E "all() - test(get_api) - kind(bin)"
-
-  clippy:
-    name: clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      # - uses: rui314/setup-mold@v1
-      - name: install mold
-        run: sudo apt-get install -y musl-tools musl-dev libssl-dev clang mold
-      - uses: moonrepo/setup-rust@v1
-        with:
-          components: clippy
-      - name: clippy
-        run: cargo clippy

.gitignore

@@ -105,5 +105,3 @@ venv.bak/
 
 /data
 /target
-Cargo.lock
-.torrent

Cargo.toml

@@ -8,7 +8,7 @@ license = "MIT OR Apache-2.0"
 anyhow = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-surrealdb = "1.5.4"
+surrealdb = "1.5"
 tokio = { version = "1.39", features = ["time"] }
 futures = "0.3"
 wikidata = "1.1"

flake.nix

@@ -0,0 +1,147 @@
+{
+  description = "Build a cargo project";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+
+    crane = {
+      url = "github:ipetkov/crane";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.rust-analyzer-src.follows = "";
+    };
+
+    flake-utils.url = "github:numtide/flake-utils";
+
+    advisory-db = {
+      url = "github:rustsec/advisory-db";
+      flake = false;
+    };
+  };
+
+  outputs = { self, nixpkgs, crane, fenix, flake-utils, advisory-db, ... }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+
+        inherit (pkgs) lib;
+
+        craneLib = crane.mkLib pkgs;
+        src = craneLib.cleanCargoSource ./.;
+
+        # Common arguments can be set here to avoid repeating them later
+        commonArgs = {
+          inherit src;
+          strictDeps = true;
+
+          buildInputs = [
+            # Add additional build inputs here
+          ] ++ lib.optionals pkgs.stdenv.isDarwin [
+            # Additional darwin specific inputs can be set here
+            pkgs.libiconv
+          ];
+
+          # Additional environment variables can be set directly
+          # MY_CUSTOM_VAR = "some value";
+        };
+
+        craneLibLLvmTools = craneLib.overrideToolchain
+          (fenix.packages.${system}.complete.withComponents [
+            "cargo"
+            "llvm-tools"
+            "rustc"
+          ]);
+
+        # Build *just* the cargo dependencies, so we can reuse
+        # all of that work (e.g. via cachix) when running in CI
+        cargoArtifacts = craneLib.buildDepsOnly commonArgs;
+
+        # Build the actual crate itself, reusing the dependency
+        # artifacts from above.
+        my-crate = craneLib.buildPackage (commonArgs // {
+          doCheck = false;
+          inherit cargoArtifacts;
+        });
+
+        # Define the Docker image build
+        dockerImage = pkgs.dockerTools.buildImage {
+          name = "wikidata-to-surrealdb";
+          tag = "latest";
+          copyToRoot = [ my-crate ];
+          config = {
+            Cmd = [ "${my-crate}/bin/wikidata-to-surrealdb" ];
+          };
+        };
+      in
+      {
+        checks = {
+          # Build the crate as part of `nix flake check` for convenience
+          inherit my-crate;
+
+          # Run clippy (and deny all warnings) on the crate source,
+          # again, reusing the dependency artifacts from above.
+          #
+          # Note that this is done as a separate derivation so that
+          # we can block the CI if there are issues here, but not
+          # prevent downstream consumers from building our crate by itself.
+          my-crate-clippy = craneLib.cargoClippy (commonArgs // {
+            inherit cargoArtifacts;
+            cargoClippyExtraArgs = "--all-targets -- --deny warnings";
+          });
+
+          # my-crate-doc = craneLib.cargoDoc (commonArgs // {
+          #   inherit cargoArtifacts;
+          # });
+
+          # Check formatting
+          my-crate-fmt = craneLib.cargoFmt {
+            inherit src;
+          };
+
+          # # Audit dependencies
+          # my-crate-audit = craneLib.cargoAudit {
+          #   inherit src advisory-db;
+          # };
+
+          # # Audit licenses
+          # my-crate-deny = craneLib.cargoDeny {
+          #   inherit src;
+          # };
+
+          # Run tests with cargo-nextest
+          # Consider setting `doCheck = false` on `my-crate` if you do not want
+          # the tests to run twice
+          my-crate-nextest = craneLib.cargoNextest (commonArgs // {
+            inherit cargoArtifacts;
+            partitions = 1;
+            partitionType = "count";
+          });
+        };
+
+        packages = {
+          default = my-crate;
+          dockerImage = dockerImage;
+        };
+
+        apps.default = flake-utils.lib.mkApp {
+          drv = my-crate;
+        };
+
+        devShells.default = craneLib.devShell {
+          # Inherit inputs from checks.
+          checks = self.checks.${system};
+
+          # Additional dev-shell environment variables can be set directly
+          # MY_CUSTOM_DEVELOPMENT_VAR = "something else";
+
+          # Extra inputs can be added here; cargo and rustc are provided by default.
+          packages = [
+            # pkgs.ripgrep
+          ];
+        };
+      });
+}