Merge pull request #5 from iofq/dev

bundle parsers with lua source

.github/workflows/main.yaml

@@ -3,19 +3,24 @@ name: CI
 on: [push, pull_request]
 
 jobs:
-  check-generated-parsers:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: DeterminateSystems/nix-installer-action@main
-      - run: mv generated.nix generated.nix.old
-      - run: nix develop --command "generate-parsers"
-      - run: diff generated.nix generated.nix.old
   build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@main
-      - run: nix run nixpkgs#nixfmt **/*.nix
+      - uses: DeterminateSystems/flakehub-cache-action@main
       - run: nix flake check
-      - run: nix build .
+      - run: nix run nixpkgs#nixfmt **/*.nix
+
+      # check generate-parsers
+      - run: mv generated.nix generated.nix.old
+      - run: nix develop --command "generate-parsers"
+      - run: diff generated.nix generated.nix.old
+
+      - run: nix build .#nvim-treesitter.withAllGrammars
+      # Ensure bundle is injected into source code
+      - run: grep -R "/nix/store" result/lua
+      # Ensure parsers & queries are linked to rtp
+      - run: |-
+          test -f result/parser/http.so
+          test -d result/queries/http

README.md

@@ -24,6 +24,8 @@ The `nixpkgs` `nvim-treesitter` plugin is not well equipped to handle the migrat
 
 ## Usage
 
+** See below if you also plan to install tree-sitter grammars **
+
 In your flake.nix:
 
 ```nix
@@ -40,6 +42,33 @@ In your flake.nix:
 
 ```
 
+## Parsers (withPlugins, withAllGrammars)
+
+`nvim-treesitter` expects all of the parsers and queries to be installed in a single directory (in a non-nix setting this would be done imperatively via `:TSInstall` into `~/.local`). To pacify it in a nix setting, `withPlugins` and `withAllGrammars` have been extended to bundle all defined parsers into a single path, and patch the nvim-treesitter `config.lua` `install_dir` setting to point directly at the bundle in the nix store.
+
+A few other neovim plugins define `nvim-treesitter` as a dependency, meaning we run the risk of having two separate copies of `nvim-treesitter` presented to Neovim which can cause issues because one copy will not be aware of your installed parsers. To fix, create an overlay like below to redefine `nvim-treesitter` and any dependent plugins in your nixpkgs set.
+
+*If you are not using any other plugins that depend on `nvim-treesitter`, you may skip this step, but it's still recommended.*
+
+```nix
+overlays = [
+  inputs.nvim-treesitter-main.overlays.default
+  (final: prev: {
+    vimPlugins = prev.vimPlugins.extend (
+      f: p: {
+        nvim-treesitter = p.nvim-treesitter.withAllGrammars; # or withPlugins...
+        # also redefine nvim-treesitter-textobjects (any other plugins that depend on nvim-treesitter)
+        nvim-treesitter-textobjects = p.nvim-treesitter-textobjects.override {
+          dependencies = [ f.nvim-treesitter ];
+        };
+      }
+    );
+  })
+];
+```
+
+If you need the unpatched `nvim-treesitter` plugin without any parsers/queries bundled, even after you overlay it, you can use the `nvim-treesitter-unwrapped` output of this overlay.
+
 ## Updating
 
 To update the list of parsers in `generated.nix`:

flake.nix

@@ -60,6 +60,7 @@
         in
         rec {
           nvim-treesitter-textobjects = pkgs'.vimPlugins.nvim-treesitter-textobjects;
+          nvim-treesitter-unwrapped = pkgs'.vimPlugins.nvim-treesitter-unwrapped;
           nvim-treesitter = pkgs'.vimPlugins.nvim-treesitter;
           default = nvim-treesitter;
         }

overlay.nix

@@ -46,40 +46,63 @@ let
   # pkgs.vimPlugins.nvim-treesitter.withAllGrammars
   withPlugins =
     f:
-    final.vimPlugins.nvim-treesitter.overrideAttrs {
-      passthru.dependencies = map grammarToPlugin (f (tree-sitter.builtGrammars // builtGrammars));
-    };
+    let
+      grammars = (f (tree-sitter.builtGrammars // builtGrammars));
+      grammarNames = lib.concatStringsSep " " (
+        map (g: builtins.elemAt (builtins.split "-" g.name) 0) grammars
+      );
+      bundle = pkgs.symlinkJoin {
+        name = "nvim-treesitter-bundle";
+        paths = map grammarToPlugin grammars;
+      };
+    in
+    final.vimPlugins.nvim-treesitter-unwrapped.overrideAttrs (old: {
+      postInstall = old.postInstall + ''
+        # ensure runtime queries get linked to RTP (:TSInstall does this too)
+        mkdir -p $out/queries
+        for grammar in ${grammarNames}; do
+            ln -sfT $src/runtime/queries/$grammar $out/queries/$grammar
+        done
+
+        # patch nvim-treesitter with parser bundle path
+        ln -sfT ${bundle}/parser $out/parser
+        substituteInPlace $out/lua/nvim-treesitter/config.lua \
+          --replace-fail "install_dir = vim.fs.joinpath(vim.fn.stdpath('data'), 'site')," \
+          "install_dir = '$out'"
+      '';
+    });
 
   withAllGrammars = withPlugins (_: allGrammars);
 in
 {
   vimPlugins = prev.vimPlugins.extend (
-    final': prev': {
-      nvim-treesitter = prev'.nvim-treesitter.overrideAttrs (old: rec {
-        src = inputs.nvim-treesitter;
-        name = "${old.pname}-${src.rev}";
-        postPatch = "";
-        # ensure runtime queries get linked to RTP (:TSInstall does this too)
-        postInstall = "
-          mkdir -p $out/queries
-          cp -a $src/runtime/queries/* $out/queries
-        ";
-        passthru = (prev'.nvim-treesitter.passthru or { }) // {
-          inherit
-            builtGrammars
-            allGrammars
-            grammarToPlugin
-            withPlugins
-            withAllGrammars
-            ;
+    final': prev': rec {
+      nvim-treesitter-unwrapped = (
+        prev'.nvim-treesitter.overrideAttrs (old: rec {
+          src = inputs.nvim-treesitter;
+          name = "${old.pname}-${src.rev}";
+          postPatch = "";
+          # ensure runtime queries get linked to RTP (:TSInstall does this too)
+          passthru = (prev'.nvim-treesitter.passthru or { }) // {
+            inherit
+              builtGrammars
+              allGrammars
+              grammarToPlugin
+              withPlugins
+              withAllGrammars
+              ;
+
+            grammarPlugins = lib.mapAttrs (_: grammarToPlugin) generatedDerivations;
+          };
+          nvimSkipModules = [ "nvim-treesitter._meta.parsers" ];
+        })
+      );
+      nvim-treesitter = nvim-treesitter-unwrapped;
 
-          grammarPlugins = lib.mapAttrs (_: grammarToPlugin) generatedDerivations;
-        };
-        nvimSkipModules = [ "nvim-treesitter._meta.parsers" ];
-      });
       nvim-treesitter-textobjects = prev'.nvim-treesitter-textobjects.overrideAttrs (old: {
         version = inputs.nvim-treesitter-textobjects.rev;
         src = inputs.nvim-treesitter-textobjects;
+        dependencies = [ final'.nvim-treesitter ];
       });
     }
   );