Speed up template migrations (#14679)

This PR does two things: - Computes UTF-16 string positions in Rust rather than in JS — eliminating a significant number of traversals of the input string - Applies replacements to the content in ascending order so we only ever move forward through the source string — this lets v8 optimize string concatenation
2026-02-01 17:26:34 +00:00 · 2024-10-16 07:13:48 -04:00 · 2024-10-16 07:13:48 -04:00 · 92a43d6904
commit 92a43d6904
parent be6c69e29f
8 changed files with 189 additions and 53 deletions
--- a/crates/node/src/lib.rs
+++ b/crates/node/src/lib.rs
@ -1,6 +1,10 @@
+use utf16::IndexConverter;
+
 #[macro_use]
 extern crate napi_derive;

+mod utf16;
+
 #[derive(Debug, Clone)]
 #[napi(object)]
 pub struct ChangedContent {
@ -123,13 +127,25 @@ impl Scanner {
    &mut self,
    input: ChangedContent,
  ) -> Vec<CandidateWithPosition> {
+    let content = input.content.unwrap_or_else(|| {
+      std::fs::read_to_string(&input.file.unwrap()).expect("Failed to read file")
+    });
+
+    let input = ChangedContent {
+      file: None,
+      content: Some(content.clone()),
+      extension: input.extension,
+    };
+
+    let mut utf16_idx = IndexConverter::new(&content[..]);
+
    self
      .scanner
      .get_candidates_with_positions(input.into())
      .into_iter()
      .map(|(candidate, position)| CandidateWithPosition {
        candidate,
-        position: position as i64,
+        position: utf16_idx.get(position),
      })
      .collect()
  }
--- a/crates/node/src/utf16.rs
+++ b/crates/node/src/utf16.rs
@ -0,0 +1,100 @@
+/// The `IndexConverter` is used to convert UTF-8 *BYTE* indexes to UTF-16
+/// *character* indexes
+#[derive(Clone)]
+pub struct IndexConverter<'a> {
+  input: &'a str,
+  curr_utf8: usize,
+  curr_utf16: usize,
+}
+
+impl<'a> IndexConverter<'a> {
+  pub fn new(input: &'a str) -> Self {
+    Self {
+      input,
+      curr_utf8: 0,
+      curr_utf16: 0,
+    }
+  }
+
+  pub fn get(&mut self, pos: usize) -> i64 {
+    #[cfg(debug_assertions)]
+    if self.curr_utf8 > self.input.len() {
+      panic!("curr_utf8 points past the end of the input string");
+    }
+
+    if pos < self.curr_utf8 {
+      self.curr_utf8 = 0;
+      self.curr_utf16 = 0;
+    }
+
+    // SAFETY: No matter what `pos` is passed into this function `curr_utf8`
+    // will only ever be incremented up to the length of the input string.
+    //
+    // This eliminates a "potential" panic that cannot actually happen
+    let slice = unsafe {
+      self.input.get_unchecked(self.curr_utf8..)
+    };
+
+    for c in slice.chars() {
+      if self.curr_utf8 >= pos {
+        break
+      }
+
+      self.curr_utf8 += c.len_utf8();
+      self.curr_utf16 += c.len_utf16();
+    }
+
+    return self.curr_utf16 as i64;
+  }
+}
+
+#[cfg(test)]
+mod test {
+  use super::*;
+  use std::collections::HashMap;
+
+  #[test]
+  fn test_index_converter() {
+    let mut converter = IndexConverter::new("Hello 🔥🥳 world!");
+
+    let map = HashMap::from([
+      // hello<space>
+      (0, 0),
+      (1, 1),
+      (2, 2),
+      (3, 3),
+      (4, 4),
+      (5, 5),
+      (6, 6),
+
+      // inside the 🔥
+      (7, 8),
+      (8, 8),
+      (9, 8),
+      (10, 8),
+
+      // inside the 🥳
+      (11, 10),
+      (12, 10),
+      (13, 10),
+      (14, 10),
+
+      // <space>world!
+      (15, 11),
+      (16, 12),
+      (17, 13),
+      (18, 14),
+      (19, 15),
+      (20, 16),
+      (21, 17),
+
+      // Past the end should return the last utf-16 character index
+      (22, 17),
+      (100, 17),
+    ]);
+
+    for (idx_utf8, idx_utf16) in map {
+      assert_eq!(converter.get(idx_utf8), idx_utf16);
+    }
+  }
+}
--- a/packages/@tailwindcss-upgrade/package.json
+++ b/packages/@tailwindcss-upgrade/package.json
@ -38,7 +38,6 @@
    "postcss-import": "^16.1.0",
    "postcss-selector-parser": "^6.1.2",
    "prettier": "^3.3.3",
-    "string-byte-slice": "^3.0.0",
    "tailwindcss": "workspace:^",
    "tree-sitter": "^0.21.1",
    "tree-sitter-typescript": "^0.23.0"
--- a/packages/@tailwindcss-upgrade/src/template/candidates.test.ts
+++ b/packages/@tailwindcss-upgrade/src/template/candidates.test.ts
@ -1,6 +1,7 @@
 import { __unstable__loadDesignSystem } from '@tailwindcss/node'
 import { describe, expect, test } from 'vitest'
-import { extractRawCandidates, printCandidate, replaceCandidateInContent } from './candidates'
+import { extractRawCandidates, printCandidate } from './candidates'
+import { spliceChangesIntoString } from './splice-changes-into-string'

 let html = String.raw

@ -66,13 +67,20 @@ test('replaces the right positions for a candidate', async () => {
    ({ rawCandidate }) => designSystem.parseCandidate(rawCandidate).length > 0,
  )!

-  expect(replaceCandidateInContent(content, 'flex', candidate.start, candidate.end))
-    .toMatchInlineSnapshot(`
+  let migrated = spliceChangesIntoString(content, [
+    {
+      start: candidate.start,
+      end: candidate.end,
+      replacement: 'flex',
+    },
+  ])
+
+  expect(migrated).toMatchInlineSnapshot(`
+    "
+        <h1>🤠👋</h1>
+        <div class="flex" />
      "
-          <h1>🤠👋</h1>
-          <div class="flex" />
-        "
-    `)
+  `)
 })

 const candidates = [
--- a/packages/@tailwindcss-upgrade/src/template/candidates.ts
+++ b/packages/@tailwindcss-upgrade/src/template/candidates.ts
@ -1,5 +1,4 @@
 import { Scanner } from '@tailwindcss/oxide'
-import stringByteSlice from 'string-byte-slice'
 import type { Candidate, Variant } from '../../../tailwindcss/src/candidate'
 import type { DesignSystem } from '../../../tailwindcss/src/design-system'

@ -139,12 +138,3 @@ function escapeArbitrary(input: string) {
    .replaceAll('_', String.raw`\_`) // Escape underscores to keep them as-is
    .replaceAll(' ', '_') // Replace spaces with underscores
 }
-
-export function replaceCandidateInContent(
-  content: string,
-  replacement: string,
-  startByte: number,
-  endByte: number,
-) {
-  return stringByteSlice(content, 0, startByte) + replacement + stringByteSlice(content, endByte)
-}
--- a/packages/@tailwindcss-upgrade/src/template/migrate.ts
+++ b/packages/@tailwindcss-upgrade/src/template/migrate.ts
@ -2,7 +2,7 @@ import fs from 'node:fs/promises'
 import path, { extname } from 'node:path'
 import type { Config } from 'tailwindcss'
 import type { DesignSystem } from '../../../tailwindcss/src/design-system'
-import { extractRawCandidates, replaceCandidateInContent } from './candidates'
+import { extractRawCandidates } from './candidates'
 import { arbitraryValueToBareValue } from './codemods/arbitrary-value-to-bare-value'
 import { automaticVarInjection } from './codemods/automatic-var-injection'
 import { bgGradient } from './codemods/bg-gradient'
@ -10,6 +10,7 @@ import { important } from './codemods/important'
 import { prefix } from './codemods/prefix'
 import { simpleLegacyClasses } from './codemods/simple-legacy-classes'
 import { variantOrder } from './codemods/variant-order'
+import { spliceChangesIntoString, type StringChange } from './splice-changes-into-string'

 export type Migration = (
  designSystem: DesignSystem,
@ -46,19 +47,23 @@ export default async function migrateContents(
 ): Promise<string> {
  let candidates = await extractRawCandidates(contents, extension)

-  // Sort candidates by starting position desc
-  candidates.sort((a, z) => z.start - a.start)
+  let changes: StringChange[] = []

-  let output = contents
  for (let { rawCandidate, start, end } of candidates) {
    let migratedCandidate = migrateCandidate(designSystem, userConfig, rawCandidate)

-    if (migratedCandidate !== rawCandidate) {
-      output = replaceCandidateInContent(output, migratedCandidate, start, end)
+    if (migratedCandidate === rawCandidate) {
+      continue
    }
+
+    changes.push({
+      start,
+      end,
+      replacement: migratedCandidate,
+    })
  }

-  return output
+  return spliceChangesIntoString(contents, changes)
 }

 export async function migrate(designSystem: DesignSystem, userConfig: Config, file: string) {
--- a/packages/@tailwindcss-upgrade/src/template/splice-changes-into-string.ts
+++ b/packages/@tailwindcss-upgrade/src/template/splice-changes-into-string.ts
@ -0,0 +1,42 @@
+export interface StringChange {
+  start: number
+  end: number
+  replacement: string
+}
+
+/**
+ * Apply the changes to the string such that a change in the length
+ * of the string does not break the indexes of the subsequent changes.
+ */
+export function spliceChangesIntoString(str: string, changes: StringChange[]) {
+  // If there are no changes, return the original string
+  if (!changes[0]) return str
+
+  // Sort all changes in order to make it easier to apply them
+  changes.sort((a, b) => {
+    return a.end - b.end || a.start - b.start
+  })
+
+  // Append original string between each chunk, and then the chunk itself
+  // This is sort of a String Builder pattern, thus creating less memory pressure
+  let result = ''
+
+  let previous = changes[0]
+
+  result += str.slice(0, previous.start)
+  result += previous.replacement
+
+  for (let i = 1; i < changes.length; ++i) {
+    let change = changes[i]
+
+    result += str.slice(previous.end, change.start)
+    result += change.replacement
+
+    previous = change
+  }
+
+  // Add leftover string from last chunk to end
+  result += str.slice(previous.end)
+
+  return result
+}
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -306,9 +306,6 @@ importers:
      prettier:
        specifier: ^3.3.3
        version: 3.3.3
-      string-byte-slice:
-        specifier: ^3.0.0
-        version: 3.0.0
      tailwindcss:
        specifier: workspace:^
        version: link:../tailwindcss
@ -1009,7 +1006,6 @@ packages:
  '@parcel/watcher-darwin-arm64@2.4.2-alpha.0':
    resolution: {integrity: sha512-2xH4Ve7OKjIh+4YRfTN3HGJa2W8KTPLOALHZj5fxcbTPwaVxdpIRItDrcikUx2u3AzGAFme7F+AZZXHnf0F15Q==}
    engines: {node: '>= 10.0.0'}
-    cpu: [arm64]
    os: [darwin]

  '@parcel/watcher-darwin-x64@2.4.1':
@ -1021,7 +1017,6 @@ packages:
  '@parcel/watcher-darwin-x64@2.4.2-alpha.0':
    resolution: {integrity: sha512-xtjmXUH4YZVah5+7Q0nb+fpRP5qZn9cFfuPuZ4k77UfUGVwhacgZyIRQgIOwMP3GkgW4TsrKQaw1KIe7L1ZqcQ==}
    engines: {node: '>= 10.0.0'}
-    cpu: [x64]
    os: [darwin]

  '@parcel/watcher-freebsd-x64@2.4.1':
@ -1045,7 +1040,6 @@ packages:
  '@parcel/watcher-linux-arm64-glibc@2.4.2-alpha.0':
    resolution: {integrity: sha512-vIIOcZf+fgsRReIK3Fw0WINvGo9UwiXfisnqYRzfpNByRZvkEPkGTIVe8iiDp72NhPTVmwIvBqM6yKDzIaw8GQ==}
    engines: {node: '>= 10.0.0'}
-    cpu: [arm64]
    os: [linux]

  '@parcel/watcher-linux-arm64-musl@2.4.1':
@ -1057,7 +1051,6 @@ packages:
  '@parcel/watcher-linux-arm64-musl@2.4.2-alpha.0':
    resolution: {integrity: sha512-gXqEAoLG9bBCbQNUgqjSOxHcjpmCZmYT9M8UvrdTMgMYgXgiWcR8igKlPRd40mCIRZSkMpN2ScSy2WjQ0bQZnQ==}
    engines: {node: '>= 10.0.0'}
-    cpu: [arm64]
    os: [linux]

  '@parcel/watcher-linux-x64-glibc@2.4.1':
@ -1069,7 +1062,6 @@ packages:
  '@parcel/watcher-linux-x64-glibc@2.4.2-alpha.0':
    resolution: {integrity: sha512-/WJJ3Y46ubwQW+Z+mzpzK3pvqn/AT7MA63NB0+k9GTLNxJQZNREensMtpJ/FJ+LVIiraEHTY22KQrsx9+DeNbw==}
    engines: {node: '>= 10.0.0'}
-    cpu: [x64]
    os: [linux]

  '@parcel/watcher-linux-x64-musl@2.4.1':
@ -1081,7 +1073,6 @@ packages:
  '@parcel/watcher-linux-x64-musl@2.4.2-alpha.0':
    resolution: {integrity: sha512-1dz4fTM5HaANk3RSRmdhALT+bNqTHawVDL1D77HwV/FuF/kSjlM3rGrJuFaCKwQ5E8CInHCcobqMN8Jh8LYaRg==}
    engines: {node: '>= 10.0.0'}
-    cpu: [x64]
    os: [linux]

  '@parcel/watcher-win32-arm64@2.4.1':
@ -1105,7 +1096,6 @@ packages:
  '@parcel/watcher-win32-x64@2.4.2-alpha.0':
    resolution: {integrity: sha512-U2abMKF7JUiIxQkos19AvTLFcnl2Xn8yIW1kzu+7B0Lux4Gkuu/BUDBroaM1s6+hwgK63NOLq9itX2Y3GwUThg==}
    engines: {node: '>= 10.0.0'}
-    cpu: [x64]
    os: [win32]

  '@parcel/watcher@2.4.1':
@ -1441,7 +1431,6 @@ packages:

  bun@1.1.29:
    resolution: {integrity: sha512-SKhpyKNZtgxrVel9ec9xon3LDv8mgpiuFhARgcJo1YIbggY2PBrKHRNiwQ6Qlb+x3ivmRurfuwWgwGexjpgBRg==}
-    cpu: [arm64, x64]
    os: [darwin, linux, win32]
    hasBin: true

@ -2210,13 +2199,11 @@ packages:
  lightningcss-darwin-arm64@1.26.0:
    resolution: {integrity: sha512-n4TIvHO1NY1ondKFYpL2ZX0bcC2y6yjXMD6JfyizgR8BCFNEeArINDzEaeqlfX9bXz73Bpz/Ow0nu+1qiDrBKg==}
    engines: {node: '>= 12.0.0'}
-    cpu: [arm64]
    os: [darwin]

  lightningcss-darwin-x64@1.26.0:
    resolution: {integrity: sha512-Rf9HuHIDi1R6/zgBkJh25SiJHF+dm9axUZW/0UoYCW1/8HV0gMI0blARhH4z+REmWiU1yYT/KyNF3h7tHyRXUg==}
    engines: {node: '>= 12.0.0'}
-    cpu: [x64]
    os: [darwin]

  lightningcss-freebsd-x64@1.26.0:
@ -2234,25 +2221,21 @@ packages:
  lightningcss-linux-arm64-gnu@1.26.0:
    resolution: {integrity: sha512-iJmZM7fUyVjH+POtdiCtExG+67TtPUTer7K/5A8DIfmPfrmeGvzfRyBltGhQz13Wi15K1lf2cPYoRaRh6vcwNA==}
    engines: {node: '>= 12.0.0'}
-    cpu: [arm64]
    os: [linux]

  lightningcss-linux-arm64-musl@1.26.0:
    resolution: {integrity: sha512-XxoEL++tTkyuvu+wq/QS8bwyTXZv2y5XYCMcWL45b8XwkiS8eEEEej9BkMGSRwxa5J4K+LDeIhLrS23CpQyfig==}
    engines: {node: '>= 12.0.0'}
-    cpu: [arm64]
    os: [linux]

  lightningcss-linux-x64-gnu@1.26.0:
    resolution: {integrity: sha512-1dkTfZQAYLj8MUSkd6L/+TWTG8V6Kfrzfa0T1fSlXCXQHrt1HC1/UepXHtKHDt/9yFwyoeayivxXAsApVxn6zA==}
    engines: {node: '>= 12.0.0'}
-    cpu: [x64]
    os: [linux]

  lightningcss-linux-x64-musl@1.26.0:
    resolution: {integrity: sha512-yX3Rk9m00JGCUzuUhFEojY+jf/6zHs3XU8S8Vk+FRbnr4St7cjyMXdNjuA2LjiT8e7j8xHRCH8hyZ4H/btRE4A==}
    engines: {node: '>= 12.0.0'}
-    cpu: [x64]
    os: [linux]

  lightningcss-win32-arm64-msvc@1.26.0:
@ -2264,7 +2247,6 @@ packages:
  lightningcss-win32-x64-msvc@1.26.0:
    resolution: {integrity: sha512-pYS3EyGP3JRhfqEFYmfFDiZ9/pVNfy8jVIYtrx9TVNusVyDK3gpW1w/rbvroQ4bDJi7grdUtyrYU6V2xkY/bBw==}
    engines: {node: '>= 12.0.0'}
-    cpu: [x64]
    os: [win32]

  lightningcss@1.26.0:
@ -2792,10 +2774,6 @@ packages:
    resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==}
    engines: {node: '>=10.0.0'}

-  string-byte-slice@3.0.0:
-    resolution: {integrity: sha512-KqTTvThKPDgBPr9jI2cOdO04tJ+upcADk4j4zmcBNmG6Bqstsq1x1Z3xvJAPqRQgPE8yocXNLVZuCfYlv4+PTg==}
-    engines: {node: '>=18.18.0'}
-
  string-width@4.2.3:
    resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
    engines: {node: '>=8'}
@ -4403,7 +4381,7 @@ snapshots:
      eslint: 8.57.0
      eslint-import-resolver-node: 0.3.9
      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
      eslint-plugin-jsx-a11y: 6.9.0(eslint@8.57.0)
      eslint-plugin-react: 7.35.0(eslint@8.57.0)
      eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0)
@ -4427,7 +4405,7 @@ snapshots:
      enhanced-resolve: 5.17.1
      eslint: 8.57.0
      eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
      fast-glob: 3.3.2
      get-tsconfig: 4.7.6
      is-core-module: 2.15.0
@ -4449,7 +4427,7 @@ snapshots:
    transitivePeerDependencies:
      - supports-color

-  eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0):
+  eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
    dependencies:
      array-includes: 3.1.8
      array.prototype.findlastindex: 1.2.5
@ -5566,8 +5544,6 @@ snapshots:

  streamsearch@1.1.0: {}

-  string-byte-slice@3.0.0: {}
-
  string-width@4.2.3:
    dependencies:
      emoji-regex: 8.0.0