feat: Initial clean release of CHM

.github/scripts/run_ci.sh

@@ -0,0 +1,383 @@
+#!/bin/bash
+
+# =============================================================================
+# Setup environment variables
+# =============================================================================
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+ROOT_DIR="$SCRIPT_DIR/../.."
+
+TRANSCRIBE_TAG_PREFIX=transcribe
+TRANSCRIBE_REPOSITORY=joplin/transcribe
+
+IS_PULL_REQUEST=0
+IS_DESKTOP_RELEASE=0
+IS_SERVER_RELEASE=0
+IS_TRANSCRIBE_RELEASE=0
+IS_LINUX=0
+IS_MACOS=0
+
+# If pull requests are coming from a branch of the main repository,
+# IS_PULL_REQUEST will be zero.
+if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
+	IS_PULL_REQUEST=1
+fi
+
+if [[ $GIT_TAG_NAME = $SERVER_TAG_PREFIX-* ]]; then
+	IS_SERVER_RELEASE=1
+fi
+
+if [[ $GIT_TAG_NAME = $TRANSCRIBE_TAG_PREFIX-* ]]; then
+	IS_TRANSCRIBE_RELEASE=1
+fi
+
+if [[ $GIT_TAG_NAME = v* ]]; then
+	IS_DESKTOP_RELEASE=1
+fi
+
+if [ "$RUNNER_OS" == "Linux" ]; then
+	IS_LINUX=1
+	IS_MACOS=0
+else
+	IS_LINUX=0
+	IS_MACOS=1
+fi
+
+DOCKER_IMAGE_PLATFORM="linux/amd64"
+
+# Tests can randomly fail in some cases, so only run them when not publishing
+# a release
+RUN_TESTS=0
+
+if [ "$IS_SERVER_RELEASE" = 0 ] && [ "$IS_DESKTOP_RELEASE" = 0 ] && [ "$IS_TRANSCRIBE_RELEASE" = 0 ]; then
+	RUN_TESTS=1
+fi
+
+if [ "$RUNNER_ARCH" == "ARM64" ]; then
+	if [ "$IS_SERVER_RELEASE" == "0" ] && [ "$IS_TRANSCRIBE_RELEASE" == "0" ]; then
+		# We exit now because nothing works properly with the ARM64 architecture.
+		# We only proceed  if building the server image.
+		echo "Running on ARM64 and not trying to build server image - early exit"
+		exit 0
+	fi
+fi
+
+if [ "$RUNNER_ARCH" == "ARM64" ]; then
+	# Canvas is only needed for tests and it doesn't build in ARM64 so remove it
+	RUN_TESTS=0
+	cd "$ROOT_DIR/packages/lib"
+	yarn remove canvas
+	cd "$ROOT_DIR"
+
+	DOCKER_IMAGE_PLATFORM="linux/arm64"
+
+	# Delete certain directories because `yarn install` will fail on ARM64.
+	rm -rf app-desktop
+	rm -rf app-mobile
+fi
+
+# =============================================================================
+# Print environment
+# =============================================================================
+
+echo "RUNNER_OS=$RUNNER_OS"
+echo "RUNNER_ARCH=$RUNNER_ARCH"
+echo "GITHUB_WORKFLOW=$GITHUB_WORKFLOW"
+echo "GITHUB_EVENT_NAME=$GITHUB_EVENT_NAME"
+echo "GITHUB_REF=$GITHUB_REF"
+echo "RUNNER_OS=$RUNNER_OS"
+echo "GIT_TAG_NAME=$GIT_TAG_NAME"
+echo "BUILD_SEQUENCIAL=$BUILD_SEQUENCIAL"
+echo "SERVER_REPOSITORY=$SERVER_REPOSITORY"
+echo "SERVER_TAG_PREFIX=$SERVER_TAG_PREFIX"
+echo "TRANSCRIBE_TAG_PREFIX=$TRANSCRIBE_TAG_PREFIX"
+echo "DOCKER_IMAGE_PLATFORM=$DOCKER_IMAGE_PLATFORM"
+
+echo "IS_CONTINUOUS_INTEGRATION=$IS_CONTINUOUS_INTEGRATION"
+echo "IS_PULL_REQUEST=$IS_PULL_REQUEST"
+echo "IS_DESKTOP_RELEASE=$IS_DESKTOP_RELEASE"
+echo "IS_SERVER_RELEASE=$IS_SERVER_RELEASE"
+echo "IS_TRANSCRIBE_RELEASE=$IS_TRANSCRIBE_RELEASE"
+echo "RUN_TESTS=$RUN_TESTS"
+echo "IS_LINUX=$IS_LINUX"
+echo "IS_MACOS=$IS_MACOS"
+
+echo "Node $( node -v )"
+echo "Npm $( npm -v )"
+echo "Yarn $( yarn -v )"
+echo "Rust $( rustc --version )"
+
+# =============================================================================
+# Install packages
+# =============================================================================
+
+cd "$ROOT_DIR"
+
+source "$(dirname "$0")/retry.sh"
+
+retry yarn install
+testResult=$?
+if [ $testResult -ne 0 ]; then
+	echo "Yarn installation failed. Search for 'exit code 1' in the log for more information."
+	exit $testResult
+fi
+
+# =============================================================================
+# Run test units
+# =============================================================================
+
+if [ "$RUN_TESTS" == "1" ]; then
+	echo "Step: Running tests..."
+
+	# On Linux, we run the Joplin Server tests using PostgreSQL
+	if [ "$IS_LINUX" == "1" ]; then
+		echo "Running Joplin Server tests using PostgreSQL..."
+		# Docker Hub auth/pulls occasionally time out on CI, so retry a few
+		# times before giving up.
+		cmdResult=1
+		for attempt in 1 2 3; do
+			sudo docker compose --parallel 1 --file docker-compose.db-dev.yml up -d
+			cmdResult=$?
+			if [ $cmdResult -eq 0 ]; then
+				break
+			fi
+			echo "docker compose up failed (attempt $attempt). Retrying in 15s..."
+			sudo docker compose --file docker-compose.db-dev.yml down --remove-orphans || true
+			sleep 15
+		done
+		if [ $cmdResult -ne 0 ]; then
+			exit $cmdResult
+		fi
+		export JOPLIN_TESTS_SERVER_DB=pg
+	else
+		echo "Running Joplin Server tests using SQLite..."
+	fi
+
+	# Need this because we're getting this error:
+	#
+	# @joplin/lib: FATAL ERROR: Ineffective mark-compacts near heap limit
+	# Allocation failed - JavaScript heap out of memory
+	#
+	# https://stackoverflow.com/questions/38558989
+	export NODE_OPTIONS="--max-old-space-size=32768"
+	yarn test-ci
+	testResult=$?
+	if [ $testResult -ne 0 ]; then
+		exit $testResult
+	fi
+
+	# Run the test files changed by the pull request several times, to catch
+	# flaky tests before they are merged.
+	if [ "$IS_PULL_REQUEST" == "1" ]; then
+		echo "Step: Checking changed tests for flakiness..."
+
+		"$SCRIPT_DIR/repeat_changed_tests.sh" "$GITHUB_BASE_REF"
+		testResult=$?
+		if [ $testResult -ne 0 ]; then
+			exit $testResult
+		fi
+	fi
+fi
+
+# =============================================================================
+# Run linter for pull requests only. We also don't want this to make the desktop
+# release randomly fail.
+# =============================================================================
+
+if [ "$RUN_TESTS" == "1" ]; then
+	echo "Step: Running linter..."
+
+	yarn linter-ci ./
+	testResult=$?
+	if [ $testResult -ne 0 ]; then
+		exit $testResult
+	fi
+
+	yarn packageJsonLint
+	testResult=$?
+	if [ $testResult -ne 0 ]; then
+		exit $testResult
+	fi
+fi
+
+# =============================================================================
+# Validate translations - this is needed as some users manually edit .po files
+# (and often make mistakes) instead of using a proper tool like poedit. Doing it
+# for Linux only is sufficient.
+# =============================================================================
+
+if [ "$IS_LINUX" == "1" ]; then
+	echo "Step: Validating translations..."
+
+	node packages/tools/validate-translation.js
+	testResult=$?
+	if [ $testResult -ne 0 ]; then
+		exit $testResult
+	fi
+fi
+
+# =============================================================================
+# Check that we didn't lose any string due to gettext not being able to parse
+# newly modified or added scripts. This is convenient to quickly view on GitHub
+# what commit may have broken translation building.
+# =============================================================================
+
+if [ "$RUN_TESTS" == "1" ]; then
+	if [ "$IS_LINUX" == "1" ]; then
+		echo "Step: Checking for lost translation strings..."
+
+		xgettext --version
+
+		node packages/tools/build-translation.js --missing-strings-check-only
+		testResult=$?
+		if [ $testResult -ne 0 ]; then
+			exit $testResult
+		fi
+	fi
+fi
+
+# =============================================================================
+# Check .gitignore and .ignore.eslint files - they should be updated when
+# new TypeScript files are added by running `yarn updateIgnored`.
+# See coding_style.md
+# =============================================================================
+
+if [ "$IS_LINUX" == "1" ]; then
+	echo "Step: Checking for files that should have been ignored..."
+
+	# .gitignore and .ignore.eslint can be modified during yarn install. Reset them
+	# so that checkIgnoredFiles works.
+	git restore .gitignore .ignore.eslint
+
+	node packages/tools/checkIgnoredFiles.js 
+	testResult=$?
+	if [ $testResult -ne 0 ]; then
+		exit $testResult
+	fi
+fi
+
+# =============================================================================
+# Check that the website still builds
+# =============================================================================
+
+if [ "$RUN_TESTS" == "1" ]; then
+	if [ "$IS_LINUX" == "1" ]; then
+		echo "Step: Check that the website still builds..."
+
+		mkdir -p ../joplin-website/docs
+		CROWDIN_PERSONAL_TOKEN="$CROWDIN_PERSONAL_TOKEN" yarn crowdinDownload
+		SKIP_SPONSOR_PROCESSING=1 yarn buildWebsite
+		testResult=$?
+		if [ $testResult -ne 0 ]; then
+			exit $testResult
+		fi
+	fi
+fi
+
+# =============================================================================
+# Spellchecking
+# =============================================================================
+
+if [ "$IS_LINUX" == "1" ]; then
+	echo "Step: Spellchecking..."
+
+	yarn spellcheck --all
+	testResult=$?
+	if [ $testResult -ne 0 ]; then
+		exit $testResult
+	fi
+fi
+
+# =============================================================================
+# Find out if we should run the build or not. Electron-builder gets stuck when
+# building PRs so we disable it in this case. The Linux build should provide
+# enough info if the app builds or not.
+# https://github.com/electron-userland/electron-builder/issues/4263
+# =============================================================================
+
+if [ "$IS_PULL_REQUEST" == "1" ]; then
+	if [ "$IS_MACOS" == "1" ]; then
+		echo "Step: Not building Electron app"
+		exit 0
+	fi
+fi
+
+# =============================================================================
+# Build the Electron app or Docker image depending on the current tag.
+#
+# If the current tag is a desktop release tag (starts with "v", such as
+# "v1.4.7"), we build and publish to GitHub. Otherwise we only build but don't
+# publish to GitHub. It helps finding out any issue in pull requests and dev
+# branch.
+# =============================================================================
+
+cd "$ROOT_DIR/packages/app-desktop"
+
+# On macOS, `yarn dist` can fail randomly with `hdiutil detach ... Resource
+# busy` when packaging the DMG, because another process (Spotlight mds,
+# diskimages-helper, etc.) still holds the mounted image. Retry the whole
+# build a couple of times to absorb the flake.
+run_yarn_dist() {
+	local attempts=3
+	local i=1
+	while true; do
+		USE_HARD_LINKS=false yarn dist "$@"
+		local result=$?
+		if [ $result -eq 0 ]; then
+			return 0
+		fi
+		if [ "$IS_MACOS" != "1" ] || [ $i -ge $attempts ]; then
+			return $result
+		fi
+		echo "yarn dist failed (attempt $i/$attempts) - retrying..."
+		i=$((i + 1))
+		sleep 10
+	done
+}
+
+if [ "$IS_DESKTOP_RELEASE" == "1" ]; then
+	echo "Step: Building and publishing desktop application..."
+	# cd "$ROOT_DIR/packages/tools"
+	# node bundleDefaultPlugins.js
+	cd "$ROOT_DIR/packages/app-desktop"
+
+	if [ "$IS_MACOS" == "1" ]; then
+		# This is to fix this error:
+		#
+		# Exit code: ENOENT. spawn /usr/bin/python ENOENT
+		#
+		# Ref: https://github.com/electron-userland/electron-builder/issues/6767#issuecomment-1096589528
+		#
+		# It can be removed once we upgrade to electron-builder@23, however we
+		# cannot currently do this due to this error:
+		# https://github.com/laurent22/joplin/issues/8149
+		#
+		# electron-builder@24, however, still expects the python binary to be named
+		# "python" and seems to no longer respect the PYTHON_PATH environment variable.
+		# We work around this by aliasing python.
+		alias python=$(which python3)
+	fi
+	run_yarn_dist
+elif [[ $IS_LINUX = 1 ]] && [ "$IS_SERVER_RELEASE" == "1" ]; then
+	echo "Step: Building Joplin Server Docker Image..."
+	cd "$ROOT_DIR"
+	yarn buildServerDocker --docker-file Dockerfile.server --platform $DOCKER_IMAGE_PLATFORM --tag-name $GIT_TAG_NAME --push-images --repository $SERVER_REPOSITORY
+elif [[ $IS_LINUX = 1 ]] && [ "$IS_TRANSCRIBE_RELEASE" == "1" ]; then
+	echo "Step: Building Joplin Transcribe Docker Image..."
+	cd "$ROOT_DIR"
+	yarn buildServerDocker --docker-file Dockerfile.transcribe --platform $DOCKER_IMAGE_PLATFORM --tag-name $GIT_TAG_NAME --push-images --repository $TRANSCRIBE_REPOSITORY
+else
+	echo "Step: Building but *not* publishing desktop application..."
+
+	if [ "$IS_MACOS" == "1" ]; then
+		# See above why we need to specify Python
+		alias python=$(which python3)
+
+		# We also want to disable signing the app in this case, because
+		# it randomly fails and we don't even need it
+		# https://www.electron.build/code-signing#how-to-disable-code-signing-during-the-build-process-on-macos
+		export CSC_IDENTITY_AUTO_DISCOVERY=false
+		npm pkg set 'build.mac.identity'=null --json
+	fi
+	run_yarn_dist --publish=never
+fi