From d88dc102128616a9d73a1a4e2e4cdf37ef264d8c Mon Sep 17 00:00:00 2001 From: Kevin Pruett Date: Mon, 29 Jun 2020 11:39:08 -0400 Subject: [PATCH] Add Algolia indexing script to CI (#9332) --- .circleci/config.yml | 304 ++++++++++++++---------- .circleci/config/jobs/algolia-index.yml | 14 ++ .circleci/config/workflows/ci.yml | 6 + website/.env | 3 + website/.gitignore | 4 + website/package-lock.json | 141 +++++++++++ website/package.json | 3 + website/scripts/index_search_content.js | 101 ++++++++ 8 files changed, 445 insertions(+), 131 deletions(-) create mode 100644 .circleci/config/jobs/algolia-index.yml create mode 100644 website/.env create mode 100644 website/scripts/index_search_content.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d3d648ebc..c7f67e89fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,35 +4,6 @@ ### *** version: 2 jobs: - pre-flight-checks: - docker: - - image: circleci/buildpack-deps - environment: - - CCI_VERSION: 0.1.5691 - shell: /usr/bin/env bash -euo pipefail - steps: - - checkout - - run: - command: | - export CCI_PATH=/tmp/circleci-cli/$CCI_VERSION - mkdir -p $CCI_PATH - NAME=circleci-cli_${CCI_VERSION}_${ARCH} - URL=$BASE/v${CCI_VERSION}/${NAME}.tar.gz - curl -sSL $URL \ - | tar --overwrite --strip-components=1 -xz -C $CCI_PATH "${NAME}/circleci" - # Add circleci to the path for subsequent steps. - echo "export PATH=$CCI_PATH:\$PATH" >> $BASH_ENV - # Done, print some debug info. - set -x - . $BASH_ENV - which circleci - circleci version - environment: - ARCH: linux_amd64 - BASE: https://github.com/CircleCI-Public/circleci-cli/releases/download - name: Install CircleCI CLI - - run: - command: make ci-verify install-ui-dependencies: docker: - environment: @@ -54,50 +25,6 @@ jobs: key: yarn-lock-v6-{{ checksum "ui/yarn.lock" }} paths: - ui/node_modules - build-go-dev: - machine: true - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /go/src/github.com/hashicorp/vault - steps: - - run: - command: | - [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; } - # Install Go - curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" - sudo rm -rf /usr/local/go - sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" - rm -f "go${GO_VERSION}.linux-amd64.tar.gz" - GOPATH="/go" - mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; } - echo "export GOPATH='$GOPATH'" >> "$BASH_ENV" - echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV" - - echo "$ go version" - go version - name: Setup Go - working_directory: ~/ - - checkout - - attach_workspace: - at: . - - run: - command: | - # Move dev UI assets to expected location - rm -rf ./pkg - mkdir ./pkg - - # Build dev binary - make ci-bootstrap dev - name: Build dev binary - - persist_to_workspace: - paths: - - bin - root: . - environment: - - CIRCLECI_CLI_VERSION: 0.1.5546 - - GO_TAGS: '' - - GO_VERSION: 1.13.8 - - GO111MODULE: 'off' - - GOTESTSUM_VERSION: 0.3.3 test-ui: docker: - environment: @@ -158,23 +85,11 @@ jobs: export PATH="${PWD}"/bin:${PATH} make test-ui-browserstack name: Run Browserstack Tests - test-go: + build-go-dev: machine: true shell: /usr/bin/env bash -euo pipefail -c working_directory: /go/src/github.com/hashicorp/vault - parallelism: 2 steps: - - run: - command: | - # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing - [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* ]] && { - # stop the job from this step - circleci-agent step halt - } - # exit with success either way - exit 0 - name: Check branch name - working_directory: ~/ - run: command: | [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; } @@ -193,59 +108,42 @@ jobs: name: Setup Go working_directory: ~/ - checkout + - attach_workspace: + at: . - run: command: | - set -x + # Move dev UI assets to expected location + rm -rf ./pkg + mkdir ./pkg - # Install CircleCI CLI - curl -sSL \ - "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \ - | sudo tar --overwrite -xz \ - -C /usr/local/bin \ - "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci" - - # Split Go tests by prior test times - package_names=$(go list \ - -tags "${GO_TAGS}" \ - ./... \ - | grep -v /integ \ - | grep -v /vendor/ \ - | sort \ - | circleci tests split --split-by=timings --timings-type=classname) - - # Install gotestsum - curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz" \ - | sudo tar --overwrite -xz -C /usr/local/bin gotestsum - - # Run tests - make prep - mkdir -p test-results/go-test - CGO_ENABLED= \ - VAULT_ADDR= \ - VAULT_TOKEN= \ - VAULT_DEV_ROOT_TOKEN_ID= \ - VAULT_ACC= \ - VAULT_TEST_LOG_DIR=/tmp/testlogs \ - gotestsum --format=short-verbose --junitfile test-results/go-test/results.xml -- \ - -tags "${GO_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - \ - ${package_names} - name: Run Go tests - no_output_timeout: 60m - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: /tmp/testlogs + # Build dev binary + make ci-bootstrap dev + name: Build dev binary + - persist_to_workspace: + paths: + - bin + root: . environment: - CIRCLECI_CLI_VERSION: 0.1.5546 - GO_TAGS: '' - GO_VERSION: 1.13.8 - GO111MODULE: 'off' - GOTESTSUM_VERSION: 0.3.3 + algolia-index: + docker: + - image: node:12 + steps: + - checkout + - run: + command: | + if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/vault.git" ]; then + echo "Not Vault OSS Repo, not indexing Algolia" + exit 0 + fi + cd website/ + npm install + node scripts/index_search_content.js + name: Push content to Algolia Index test-go-race: machine: true shell: /usr/bin/env bash -euo pipefail -c @@ -345,6 +243,123 @@ jobs: docker push hashicorp/vault-website fi name: Build Docker Image if Necessary + test-go: + machine: true + shell: /usr/bin/env bash -euo pipefail -c + working_directory: /go/src/github.com/hashicorp/vault + parallelism: 2 + steps: + - run: + command: | + # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing + [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* ]] && { + # stop the job from this step + circleci-agent step halt + } + # exit with success either way + exit 0 + name: Check branch name + working_directory: ~/ + - run: + command: | + [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; } + # Install Go + curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" + sudo rm -rf /usr/local/go + sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" + rm -f "go${GO_VERSION}.linux-amd64.tar.gz" + GOPATH="/go" + mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; } + echo "export GOPATH='$GOPATH'" >> "$BASH_ENV" + echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV" + + echo "$ go version" + go version + name: Setup Go + working_directory: ~/ + - checkout + - run: + command: | + set -x + + # Install CircleCI CLI + curl -sSL \ + "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \ + | sudo tar --overwrite -xz \ + -C /usr/local/bin \ + "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci" + + # Split Go tests by prior test times + package_names=$(go list \ + -tags "${GO_TAGS}" \ + ./... \ + | grep -v /integ \ + | grep -v /vendor/ \ + | sort \ + | circleci tests split --split-by=timings --timings-type=classname) + + # Install gotestsum + curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz" \ + | sudo tar --overwrite -xz -C /usr/local/bin gotestsum + + # Run tests + make prep + mkdir -p test-results/go-test + CGO_ENABLED= \ + VAULT_ADDR= \ + VAULT_TOKEN= \ + VAULT_DEV_ROOT_TOKEN_ID= \ + VAULT_ACC= \ + VAULT_TEST_LOG_DIR=/tmp/testlogs \ + gotestsum --format=short-verbose --junitfile test-results/go-test/results.xml -- \ + -tags "${GO_TAGS}" \ + -timeout=60m \ + -parallel=20 \ + \ + ${package_names} + name: Run Go tests + no_output_timeout: 60m + - store_artifacts: + path: test-results + - store_test_results: + path: test-results + - store_artifacts: + path: /tmp/testlogs + environment: + - CIRCLECI_CLI_VERSION: 0.1.5546 + - GO_TAGS: '' + - GO_VERSION: 1.13.8 + - GO111MODULE: 'off' + - GOTESTSUM_VERSION: 0.3.3 + pre-flight-checks: + docker: + - image: circleci/buildpack-deps + environment: + - CCI_VERSION: 0.1.5691 + shell: /usr/bin/env bash -euo pipefail + steps: + - checkout + - run: + command: | + export CCI_PATH=/tmp/circleci-cli/$CCI_VERSION + mkdir -p $CCI_PATH + NAME=circleci-cli_${CCI_VERSION}_${ARCH} + URL=$BASE/v${CCI_VERSION}/${NAME}.tar.gz + curl -sSL $URL \ + | tar --overwrite --strip-components=1 -xz -C $CCI_PATH "${NAME}/circleci" + # Add circleci to the path for subsequent steps. + echo "export PATH=$CCI_PATH:\$PATH" >> $BASH_ENV + # Done, print some debug info. + set -x + . $BASH_ENV + which circleci + circleci version + environment: + ARCH: linux_amd64 + BASE: https://github.com/CircleCI-Public/circleci-cli/releases/download + name: Install CircleCI CLI + - run: + command: make ci-verify workflows: ci: jobs: @@ -383,6 +398,12 @@ workflows: only: - master context: vault-docs + - algolia-index: + filters: + branches: + only: + - stable-website + context: vault-docs version: 2 # Original config.yml file: @@ -513,6 +534,21 @@ workflows: # shell: /usr/bin/env bash -euo pipefail -c # working_directory: /go/src/github.com/hashicorp/vault # jobs: +# algolia-index: +# docker: +# - image: node:12 +# steps: +# - checkout +# - run: +# command: | +# if [ \"$CIRCLE_REPOSITORY_URL\" != \"git@github.com:hashicorp/vault.git\" ]; then +# echo \"Not Vault OSS Repo, not indexing Algolia\" +# exit 0 +# fi +# cd website/ +# npm install +# node scripts/index_search_content.js +# name: Push content to Algolia Index # build-go-dev: # executor: go-machine # steps: @@ -718,4 +754,10 @@ workflows: # filters: # branches: # only: -# - master \ No newline at end of file +# - master +# - algolia-index: +# context: vault-docs +# filters: +# branches: +# only: +# - stable-website \ No newline at end of file diff --git a/.circleci/config/jobs/algolia-index.yml b/.circleci/config/jobs/algolia-index.yml new file mode 100644 index 0000000000..d0218117db --- /dev/null +++ b/.circleci/config/jobs/algolia-index.yml @@ -0,0 +1,14 @@ +docker: + - image: node:12 +steps: + - checkout + - run: + name: Push content to Algolia Index + command: | + if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/vault.git" ]; then + echo "Not Vault OSS Repo, not indexing Algolia" + exit 0 + fi + cd website/ + npm install + node scripts/index_search_content.js diff --git a/.circleci/config/workflows/ci.yml b/.circleci/config/workflows/ci.yml index b49d750a31..66a891df42 100644 --- a/.circleci/config/workflows/ci.yml +++ b/.circleci/config/workflows/ci.yml @@ -36,3 +36,9 @@ jobs: branches: only: - master + - algolia-index: + context: vault-docs + filters: + branches: + only: + - stable-website diff --git a/website/.env b/website/.env new file mode 100644 index 0000000000..1a40450b0d --- /dev/null +++ b/website/.env @@ -0,0 +1,3 @@ +NEXT_PUBLIC_ALGOLIA_APP_ID=YY0FFNI7MF +NEXT_PUBLIC_ALGOLIA_INDEX=poc_VAULT +NEXT_PUBLIC_ALGOLIA_SEARCH_ONLY_API_KEY=5037da4824714676226913c65e961ca0 diff --git a/website/.gitignore b/website/.gitignore index 1d23ce1de1..f93e0e4033 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -3,3 +3,7 @@ node_modules .next out .mdx-data + +# As per Next.js conventions (https://nextjs.org/docs/basic-features/environment-variables#default-environment-variables) +.env*.local +!.env* diff --git a/website/package-lock.json b/website/package-lock.json index 97001dca03..ef4ed97484 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -4,6 +4,121 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@algolia/cache-browser-local-storage": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.3.0.tgz", + "integrity": "sha512-91Cf3IPUk84PF2wvR8ys8XO42FqaJEtIh/dyR0WvwMdv0x13GORkAvoBJgkFI2wofZqUY86jNimvHWfsWzPQ+g==", + "requires": { + "@algolia/cache-common": "4.3.0" + } + }, + "@algolia/cache-common": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.3.0.tgz", + "integrity": "sha512-AHTbOn9lk0f5IkjssXXmDgnaZfsUJVZ61sqOH1W3LyJdAscDzCj0KtwijELn8FHlLXQak7+K93/O3Oct0uHncQ==" + }, + "@algolia/cache-in-memory": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.3.0.tgz", + "integrity": "sha512-8BZS5IFEtiSFkA6vNQUXJXIWABDbSanQdkGX5LArlhbCjuykZqF68yaCjXWG10EZTySnkZLmKc+5ozYVOktJaQ==", + "requires": { + "@algolia/cache-common": "4.3.0" + } + }, + "@algolia/client-account": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.3.0.tgz", + "integrity": "sha512-8LJSvWooc+fe+XZXeu+h4dhpo9lsu3sb7rV9cpPhymYSHgEJAHaDkZEcPM1u/PBMvFe0mZXaW6nabeb3jeIRcw==", + "requires": { + "@algolia/client-common": "4.3.0", + "@algolia/client-search": "4.3.0", + "@algolia/transporter": "4.3.0" + } + }, + "@algolia/client-analytics": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.3.0.tgz", + "integrity": "sha512-BFH4ddyrqI2pE3bUctn5KtJgYqgvO0Ap9vJEHBNj6mjSKqFbTnZeVEPG3yWrOuWRCqPHR3ewcWRisNwJHG3+Mw==", + "requires": { + "@algolia/client-common": "4.3.0", + "@algolia/client-search": "4.3.0", + "@algolia/requester-common": "4.3.0", + "@algolia/transporter": "4.3.0" + } + }, + "@algolia/client-common": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.3.0.tgz", + "integrity": "sha512-8Ohj6zXZkpwDKc8ZWVTZo2wPO4+LT5D258suGg/C6nh4UxOrFOp6QaqeQo8JZ1eqMqtfb3zv5SHgW4fZ00NCLQ==", + "requires": { + "@algolia/requester-common": "4.3.0", + "@algolia/transporter": "4.3.0" + } + }, + "@algolia/client-recommendation": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/client-recommendation/-/client-recommendation-4.3.0.tgz", + "integrity": "sha512-jCMIAWPA2hsxc5CCtoTtQAcohaG+10CxXK122Tc47t4w1K8qzSJnCjC2cHvM4UNJO+k7NrmjOYW0EXp9RKc7SQ==", + "requires": { + "@algolia/client-common": "4.3.0", + "@algolia/requester-common": "4.3.0", + "@algolia/transporter": "4.3.0" + } + }, + "@algolia/client-search": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.3.0.tgz", + "integrity": "sha512-KCgcIsNMW1/0F5OILiFTddbTAKduJHRvXQS4NxY1H9gQWMTVeWJS7VZQ/ukKBiUMLatwUQHJz2qpYm9fmqOjkQ==", + "requires": { + "@algolia/client-common": "4.3.0", + "@algolia/requester-common": "4.3.0", + "@algolia/transporter": "4.3.0" + } + }, + "@algolia/logger-common": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.3.0.tgz", + "integrity": "sha512-vQ+aukjZkRAyO9iyINBefT366UtF/B9QoA1Kw8PlY67T6fYmklFgYp3LNH/e7h/gz0py5LYY/HIwSsaTKk8/VQ==" + }, + "@algolia/logger-console": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.3.0.tgz", + "integrity": "sha512-7pWtcv1cSSa7F48gRBOZLcEWN073+WbnKjbpRrIGej+abZppw/h+22jtVZZORC8EIjFffGqz2/2e6bZiX+Jg7A==", + "requires": { + "@algolia/logger-common": "4.3.0" + } + }, + "@algolia/requester-browser-xhr": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.3.0.tgz", + "integrity": "sha512-CpUwgQhXZsnZmjEd5DTwQv1BKQNCt83bzyVdUqvljsFxZOsNQacS6lOYs0B1eD18tKHCwVMuwbYqTaLPGBXTKQ==", + "requires": { + "@algolia/requester-common": "4.3.0" + } + }, + "@algolia/requester-common": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.3.0.tgz", + "integrity": "sha512-1v73KyspJBiTzfyXupjHxikxTYjh5MoxI6mOIvAtQxRqc4ehUPAEdPCNHEvvLiCK96iKWzZaULmV0U7pj3yvTw==" + }, + "@algolia/requester-node-http": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.3.0.tgz", + "integrity": "sha512-Hg9Y8sUeSGQgoO1FpoL5jbkDzCtXI/8HXHybU6bimsX93DAz3HZWaoQFKmIpQDNhQ8G9FLgAtzDAxS6eckDxzg==", + "requires": { + "@algolia/requester-common": "4.3.0" + } + }, + "@algolia/transporter": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.3.0.tgz", + "integrity": "sha512-BTKHAtdQdfOJ0xzZkiyEK/2QVQJTiVgBZlOBfXp2gBtztjV26OqfW4n6Xz0o7eBRzLEwY1ot3mHF5QIVUjAsMg==", + "requires": { + "@algolia/cache-common": "4.3.0", + "@algolia/logger-common": "4.3.0", + "@algolia/requester-common": "4.3.0" + } + }, "@ampproject/toolbox-core": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/toolbox-core/-/toolbox-core-2.3.0.tgz", @@ -2145,6 +2260,27 @@ "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.4.1.tgz", "integrity": "sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ==" }, + "algoliasearch": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.3.0.tgz", + "integrity": "sha512-H2woXyqmd1nFYDrQKLZXgghNkLBTcBXJ7Q/bxQ+F9WWS4H0Kb7IlQvNi7bDzHyldhDhIthImaUwcKqr5iiyMFQ==", + "requires": { + "@algolia/cache-browser-local-storage": "4.3.0", + "@algolia/cache-common": "4.3.0", + "@algolia/cache-in-memory": "4.3.0", + "@algolia/client-account": "4.3.0", + "@algolia/client-analytics": "4.3.0", + "@algolia/client-common": "4.3.0", + "@algolia/client-recommendation": "4.3.0", + "@algolia/client-search": "4.3.0", + "@algolia/logger-common": "4.3.0", + "@algolia/logger-console": "4.3.0", + "@algolia/requester-browser-xhr": "4.3.0", + "@algolia/requester-common": "4.3.0", + "@algolia/requester-node-http": "4.3.0", + "@algolia/transporter": "4.3.0" + } + }, "alphanum-sort": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz", @@ -4799,6 +4935,11 @@ } } }, + "dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" + }, "download": { "version": "6.2.5", "resolved": "https://registry.npmjs.org/download/-/download-6.2.5.tgz", diff --git a/website/package.json b/website/package.json index e16c919f77..c7bae87001 100644 --- a/website/package.json +++ b/website/package.json @@ -24,7 +24,10 @@ "@hashicorp/react-text-and-content": "^4.1.0", "@hashicorp/react-use-cases": "^1.0.4", "@hashicorp/react-vertical-text-block-list": "^2.0.1", + "algoliasearch": "^4.3.0", "babel-plugin-import-glob-array": "^0.2.0", + "dotenv": "^8.2.0", + "gray-matter": "^4.0.2", "imagemin-mozjpeg": "^8.0.0", "imagemin-optipng": "^7.1.0", "imagemin-svgo": "^7.1.0", diff --git a/website/scripts/index_search_content.js b/website/scripts/index_search_content.js new file mode 100644 index 0000000000..871b542dcf --- /dev/null +++ b/website/scripts/index_search_content.js @@ -0,0 +1,101 @@ +require('dotenv').config() + +const algoliasearch = require('algoliasearch') +const glob = require('glob') +const matter = require('gray-matter') +const path = require('path') + +// In addition to the content of the page, +// define additional front matter attributes that will be search-indexable +const SEARCH_DIMENSIONS = ['page_title', 'description'] + +main() + +async function main() { + const pagesFolder = path.join(__dirname, '../pages') + + // Grab all search-indexable content and format for Algolia + const searchObjects = glob + .sync(path.join(pagesFolder, '**/*.mdx')) + .map((fullPath) => { + const { content, data } = matter.read(fullPath) + + // Get path relative to `pages` + const __resourcePath = fullPath.replace(`${pagesFolder}/`, '') + + // Use clean URL for Algolia id + const objectID = __resourcePath.replace('.mdx', '') + + const searchableDimensions = Object.keys(data) + .filter((key) => SEARCH_DIMENSIONS.includes(key)) + .map((dimension) => ({ + [dimension]: data[dimension], + })) + + return { + ...searchableDimensions, + content, + __resourcePath, + objectID, + } + }) + + try { + await indexSearchContent(searchObjects) + } catch (e) { + console.error(e) + process.exit(1) + } +} + +async function indexSearchContent(objects) { + const { + NEXT_PUBLIC_ALGOLIA_APP_ID: appId, + NEXT_PUBLIC_ALGOLIA_INDEX: index, + ALGOLIA_API_KEY: apiKey, + } = process.env + + if (!apiKey || !appId || !index) { + throw new Error( + `[*** Algolia Search Indexing Error ***] Received: ALGOLIA_API_KEY=${apiKey} ALGOLIA_APP_ID=${appId} ALGOLIA_INDEX=${index} \n Please ensure all Algolia Search-related environment vars are set in CI settings.` + ) + } + + console.log(`updating ${objects.length} indices...`) + + try { + const searchClient = algoliasearch(appId, apiKey) + const searchIndex = searchClient.initIndex(index) + + await searchIndex.partialUpdateObjects(objects, { + createIfNotExists: true, + }) + + // Remove indices for items that aren't included in the new batch + const newObjectIds = objects.map(({ objectID }) => objectID) + let staleObjects = [] + + await searchIndex.browseObjects({ + query: '', + batch: (batch) => { + staleObjects = staleObjects.concat( + batch.filter(({ objectID }) => !newObjectIds.includes(objectID)) + ) + }, + }) + + const staleIds = staleObjects.map(({ objectID }) => objectID) + + if (staleIds.length > 0) { + console.log(`deleting ${staleIds.length} stale indices:`) + console.log(staleIds) + + await searchIndex.deleteObjects(staleIds) + } + + console.log('done') + process.exit(0) + } catch (error) { + throw new Error(error) + } +}