docs: add docs sync script (#12638)

* feat: add sync scripts

* feat: edit workflows file

.github/workflows/publish-docs.yml

@@ -1,20 +0,0 @@
-# main only
-
-name: Publish docs
-
-on:
-  push:
-    branches: [main]
-
-jobs:
-  docs-menu:
-    name: Publish the docs menu
-    runs-on: ubuntu-latest
-    timeout-minutes: 3 # Default is 360
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_ASSETS_AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_ASSETS_AWS_SECRET_ACCESS_KEY }}
-    steps:
-      - uses: actions/checkout@v3
-      # Upload menu.json with a cache of 60 seconds (instead of the default 24h on the CloudFront distribution)
-      - run: 'aws s3 cp docs/menu.json s3://assets.public.serverless/website/framework/docs/menu.json --cache-control max-age=60 --region us-east-2'

.github/workflows/sync-docs.yml

@@ -0,0 +1,85 @@
+# This workflow is responsible for syncing the docs menu and content to Dev and Prod stages with Algolia and ChatGPT
+
+name: Sync Docs
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    paths:
+      - 'docs/**'
+  workflow_dispatch:
+
+# This workflow contains three jobs
+jobs:
+  # Publish docs menu
+  docs-menu:
+    name: Publish the docs menu
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
+    timeout-minutes: 3 # Default is 360
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_ASSETS_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_ASSETS_AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v3
+      # Upload menu.json with a cache of 60 seconds (instead of the default 24h on the CloudFront distribution)
+      - run: 'aws s3 cp docs/menu.json s3://assets.public.serverless/website/framework/docs/menu.json --cache-control max-age=60 --region us-east-2'
+
+  # Sync docs in Prod
+  sync-docs-prod:
+    name: Sync Docs (Prod)
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run sync script
+        env:
+          ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+          ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY_PROD }}
+          ALGOLIA_DOCS_INDEX: ${{ secrets.ALGOLIA_DOCS_INDEX_PROD }}
+        run: node scripts/sync-docs.js
+
+  # Sync docs in Dev (push to docs-dev branch)
+  sync-docs-dev:
+    name: Sync Docs (Dev)
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Git
+        run: |
+          git config --global user.name 'github-actions[bot]'
+          git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+
+      - name: Push changes to docs-dev
+        run: |
+          git checkout -b docs-dev
+          git push origin docs-dev --force
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run sync script for dev
+        env:
+          ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+          ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY_DEV }}
+          ALGOLIA_DOCS_INDEX: ${{ secrets.ALGOLIA_DOCS_INDEX_DEV }}
+        run: node scripts/sync-docs.js

package.json

@@ -96,6 +96,7 @@
     "@serverless/test": "^11.1.1",
     "@serverlessinc/standards": "*",
     "adm-zip": "^0.5.10",
+    "algoliasearch": "^4.23.3",
     "aws4": "^1.12.0",
     "chai": "^4.3.7",
     "chai-as-promised": "^7.1.1",
@@ -103,11 +104,13 @@
     "eslint": "^8.57.0",
     "git-list-updated": "^1.2.1",
     "github-release-from-cc-changelog": "^2.3.0",
+    "gray-matter": "^4.0.3",
     "husky": "^4.3.8",
     "jszip": "^3.10.1",
     "lint-staged": "^13.2.2",
     "log": "^6.3.1",
     "log-node": "^8.0.3",
+    "marked": "^13.0.0",
     "mocha": "^9.2.2",
     "mock-require": "^3.0.3",
     "ncjsm": "^4.3.2",
@@ -117,6 +120,7 @@
     "sinon": "^13.0.2",
     "sinon-chai": "^3.7.0",
     "standard-version": "^9.5.0",
+    "striptags": "^3.2.0",
     "tsx": "^4.15.6",
     "xml2js": "^0.4.23"
   },

scripts/sync-docs.js

@@ -0,0 +1,117 @@
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import { marked } from 'marked'
+import striptags from 'striptags'
+import grayMatter from 'gray-matter'
+import algoliasearch from 'algoliasearch'
+
+// Environment variables
+const { ALGOLIA_API_KEY, ALGOLIA_APP_ID, ALGOLIA_DOCS_INDEX } = process.env
+
+// Initialize Algolia client and index
+const algoliaClient = algoliasearch(ALGOLIA_APP_ID, ALGOLIA_API_KEY)
+const algoliaIndex = algoliaClient.initIndex(ALGOLIA_DOCS_INDEX)
+
+// Define __dirname for ESM
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+// Function to fix frontmatter in markdown files
+const fixFrontmatter = (file) => {
+  if (file && typeof file === 'string') {
+    return file.replace('<!--', '---').replace('-->', '---')
+  }
+  return file
+}
+
+// Function to replace all items in Algolia index
+const replaceAllAlgoliaItems = async (items, options = {}) => {
+  try {
+    await algoliaIndex.replaceAllObjects(items, options)
+    console.log('Replaced all Algolia items successfully')
+  } catch (error) {
+    console.error('Error replacing Algolia items:', error)
+    throw error
+  }
+}
+
+// Function to remove a specific line from markdown content
+const removeSpecificLine = (markdownContent) => {
+  return markdownContent.replace(
+    /### \[Read this on the main serverless docs site\]\(.*\)/,
+    '',
+  )
+}
+
+// Function to preprocess markdown content
+const preprocessMarkdown = (markdownContent) => {
+  const cleanedMarkdown = removeSpecificLine(markdownContent)
+  const htmlContent = marked(cleanedMarkdown)
+  return striptags(htmlContent)
+}
+
+// Function to traverse the repository and collect markdown files
+const traverseRepo = (dir) => {
+  const files = []
+  const items = fs.readdirSync(dir)
+  items.forEach((item) => {
+    const fullPath = path.join(dir, item)
+    const stat = fs.statSync(fullPath)
+
+    if (stat.isDirectory()) {
+      files.push(...traverseRepo(fullPath))
+    } else if (stat.isFile() && item.endsWith('.md')) {
+      files.push({
+        name: item,
+        path: fullPath,
+      })
+    }
+  })
+
+  return files
+}
+
+// Function to fetch file content locally
+const getFileContent = (filePath) => {
+  return fs.readFileSync(filePath, 'utf-8')
+}
+
+// Function to synchronize documents between the local filesystem and Algolia
+const syncWithAlgolia = async () => {
+  try {
+    const docsDir = path.join(__dirname, '../docs')
+    const files = traverseRepo(docsDir)
+
+    console.log(`Found ${files.length} local files`)
+
+    const itemsToUpdate = await Promise.all(
+      files.map(async (file) => {
+        const fileContent = getFileContent(file.path)
+        const fixedFile = fixFrontmatter(fileContent)
+        const { data: frontmatter, content: markdownContent } =
+          grayMatter(fixedFile)
+        const { title = file.name, description } = frontmatter || {}
+        const content = preprocessMarkdown(markdownContent)
+
+        const objectID = path.relative(docsDir, file.path).replace('.md', '')
+
+        return {
+          objectID,
+          title,
+          description,
+          content,
+          githubFilePath: file.path,
+        }
+      }),
+    )
+
+    await replaceAllAlgoliaItems(itemsToUpdate)
+    console.log('Sync with Algolia completed successfully')
+  } catch (error) {
+    console.error('Error syncing with Algolia:', error)
+  }
+}
+
+// Usage
+syncWithAlgolia()