From 60142a2d1024a1f275d91ca1bf033507ff156e2e Mon Sep 17 00:00:00 2001 From: Walid Elnozahy Date: Wed, 26 Jun 2024 16:56:42 +0300 Subject: [PATCH] docs: add docs sync script (#12638) * feat: add sync scripts * feat: edit workflows file --- .github/workflows/publish-docs.yml | 20 ----- .github/workflows/sync-docs.yml | 85 +++++++++++++++++++++ package.json | 4 + scripts/sync-docs.js | 117 +++++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 20 deletions(-) delete mode 100644 .github/workflows/publish-docs.yml create mode 100644 .github/workflows/sync-docs.yml create mode 100644 scripts/sync-docs.js diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml deleted file mode 100644 index 77a614b6c..000000000 --- a/.github/workflows/publish-docs.yml +++ /dev/null @@ -1,20 +0,0 @@ -# main only - -name: Publish docs - -on: - push: - branches: [main] - -jobs: - docs-menu: - name: Publish the docs menu - runs-on: ubuntu-latest - timeout-minutes: 3 # Default is 360 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_ASSETS_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_ASSETS_AWS_SECRET_ACCESS_KEY }} - steps: - - uses: actions/checkout@v3 - # Upload menu.json with a cache of 60 seconds (instead of the default 24h on the CloudFront distribution) - - run: 'aws s3 cp docs/menu.json s3://assets.public.serverless/website/framework/docs/menu.json --cache-control max-age=60 --region us-east-2' diff --git a/.github/workflows/sync-docs.yml b/.github/workflows/sync-docs.yml new file mode 100644 index 000000000..3fe88f718 --- /dev/null +++ b/.github/workflows/sync-docs.yml @@ -0,0 +1,85 @@ +# This workflow is responsible for syncing the docs menu and content to Dev and Prod stages with Algolia and ChatGPT + +name: Sync Docs + +on: + push: + branches: [main] + pull_request: + paths: + - 'docs/**' + workflow_dispatch: + +# This workflow contains three jobs +jobs: + # Publish docs menu + docs-menu: + name: Publish the docs menu + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} + timeout-minutes: 3 # Default is 360 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_ASSETS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_ASSETS_AWS_SECRET_ACCESS_KEY }} + steps: + - uses: actions/checkout@v3 + # Upload menu.json with a cache of 60 seconds (instead of the default 24h on the CloudFront distribution) + - run: 'aws s3 cp docs/menu.json s3://assets.public.serverless/website/framework/docs/menu.json --cache-control max-age=60 --region us-east-2' + + # Sync docs in Prod + sync-docs-prod: + name: Sync Docs (Prod) + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install dependencies + run: npm install + + - name: Run sync script + env: + ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} + ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY_PROD }} + ALGOLIA_DOCS_INDEX: ${{ secrets.ALGOLIA_DOCS_INDEX_PROD }} + run: node scripts/sync-docs.js + + # Sync docs in Dev (push to docs-dev branch) + sync-docs-dev: + name: Sync Docs (Dev) + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Git + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + - name: Push changes to docs-dev + run: | + git checkout -b docs-dev + git push origin docs-dev --force + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install dependencies + run: npm install + + - name: Run sync script for dev + env: + ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} + ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY_DEV }} + ALGOLIA_DOCS_INDEX: ${{ secrets.ALGOLIA_DOCS_INDEX_DEV }} + run: node scripts/sync-docs.js diff --git a/package.json b/package.json index a425063c6..1cde277b1 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "@serverless/test": "^11.1.1", "@serverlessinc/standards": "*", "adm-zip": "^0.5.10", + "algoliasearch": "^4.23.3", "aws4": "^1.12.0", "chai": "^4.3.7", "chai-as-promised": "^7.1.1", @@ -103,11 +104,13 @@ "eslint": "^8.57.0", "git-list-updated": "^1.2.1", "github-release-from-cc-changelog": "^2.3.0", + "gray-matter": "^4.0.3", "husky": "^4.3.8", "jszip": "^3.10.1", "lint-staged": "^13.2.2", "log": "^6.3.1", "log-node": "^8.0.3", + "marked": "^13.0.0", "mocha": "^9.2.2", "mock-require": "^3.0.3", "ncjsm": "^4.3.2", @@ -117,6 +120,7 @@ "sinon": "^13.0.2", "sinon-chai": "^3.7.0", "standard-version": "^9.5.0", + "striptags": "^3.2.0", "tsx": "^4.15.6", "xml2js": "^0.4.23" }, diff --git a/scripts/sync-docs.js b/scripts/sync-docs.js new file mode 100644 index 000000000..72f0be0b8 --- /dev/null +++ b/scripts/sync-docs.js @@ -0,0 +1,117 @@ +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' +import { marked } from 'marked' +import striptags from 'striptags' +import grayMatter from 'gray-matter' +import algoliasearch from 'algoliasearch' + +// Environment variables +const { ALGOLIA_API_KEY, ALGOLIA_APP_ID, ALGOLIA_DOCS_INDEX } = process.env + +// Initialize Algolia client and index +const algoliaClient = algoliasearch(ALGOLIA_APP_ID, ALGOLIA_API_KEY) +const algoliaIndex = algoliaClient.initIndex(ALGOLIA_DOCS_INDEX) + +// Define __dirname for ESM +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) + +// Function to fix frontmatter in markdown files +const fixFrontmatter = (file) => { + if (file && typeof file === 'string') { + return file.replace('', '---') + } + return file +} + +// Function to replace all items in Algolia index +const replaceAllAlgoliaItems = async (items, options = {}) => { + try { + await algoliaIndex.replaceAllObjects(items, options) + console.log('Replaced all Algolia items successfully') + } catch (error) { + console.error('Error replacing Algolia items:', error) + throw error + } +} + +// Function to remove a specific line from markdown content +const removeSpecificLine = (markdownContent) => { + return markdownContent.replace( + /### \[Read this on the main serverless docs site\]\(.*\)/, + '', + ) +} + +// Function to preprocess markdown content +const preprocessMarkdown = (markdownContent) => { + const cleanedMarkdown = removeSpecificLine(markdownContent) + const htmlContent = marked(cleanedMarkdown) + return striptags(htmlContent) +} + +// Function to traverse the repository and collect markdown files +const traverseRepo = (dir) => { + const files = [] + const items = fs.readdirSync(dir) + items.forEach((item) => { + const fullPath = path.join(dir, item) + const stat = fs.statSync(fullPath) + + if (stat.isDirectory()) { + files.push(...traverseRepo(fullPath)) + } else if (stat.isFile() && item.endsWith('.md')) { + files.push({ + name: item, + path: fullPath, + }) + } + }) + + return files +} + +// Function to fetch file content locally +const getFileContent = (filePath) => { + return fs.readFileSync(filePath, 'utf-8') +} + +// Function to synchronize documents between the local filesystem and Algolia +const syncWithAlgolia = async () => { + try { + const docsDir = path.join(__dirname, '../docs') + const files = traverseRepo(docsDir) + + console.log(`Found ${files.length} local files`) + + const itemsToUpdate = await Promise.all( + files.map(async (file) => { + const fileContent = getFileContent(file.path) + const fixedFile = fixFrontmatter(fileContent) + const { data: frontmatter, content: markdownContent } = + grayMatter(fixedFile) + const { title = file.name, description } = frontmatter || {} + const content = preprocessMarkdown(markdownContent) + + const objectID = path.relative(docsDir, file.path).replace('.md', '') + + return { + objectID, + title, + description, + content, + githubFilePath: file.path, + } + }), + ) + + await replaceAllAlgoliaItems(itemsToUpdate) + console.log('Sync with Algolia completed successfully') + } catch (error) { + console.error('Error syncing with Algolia:', error) + } +} + +// Usage +syncWithAlgolia()