Makefile

# Put targets here if there is a risk that a target name might conflict with a filename.
# this list is probably overkill right now.
# See: https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
.PHONY: test test-unit test-e2e test-lseval-periodic test-lseval-troubleshooting test-cluster-updates images run format verify verify-hermetic-requirements get-embeddings get-embeddings-byok get-embeddings-okp tls-scan

export PATH := $(HOME)/.local/bin:$(PATH)

ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results)
TEST_TAGS := $(if $(TEST_TAGS),$(TEST_TAGS),"")
SUITE_ID := $(if $(SUITE_ID),$(SUITE_ID),"nosuite")
PROVIDER := $(if $(PROVIDER),$(PROVIDER),"openai")
MODEL := $(if $(MODEL),$(MODEL),"gpt-5.4-mini")
OLS_CONFIG_SUFFIX := $(if $(OLS_CONFIG_SUFFIX),$(OLS_CONFIG_SUFFIX),"default")
PATH_TO_PLANTUML := ~/bin

# Python registry to where the package should be uploaded
PYTHON_REGISTRY = testpypi

default: help

images: ## Build container images
	scripts/build-container.sh

install-tools:	install-woke ## Install required utilities/tools
	@command -v uv > /dev/null || { echo >&2 "uv is not installed. Installing..."; curl -LsSf https://astral.sh/uv/install.sh | sh; }
	uv --version
	# install all dependencies, including devel ones
	uv sync --group dev --extra evaluation
	# check that correct mypy version is installed
	uv run mypy --version
	# check that correct Black version is installed
	uv run black --version
	# check that correct Ruff version is installed
	uv run ruff --version
	# check that correct Pydocstyle version is installed
	uv run pydocstyle --version


install-woke: ## Install woke, required for Inclusive Naming scan
	@command -v ./woke > /dev/null || { echo >&2 "woke is not installed. Installing..."; curl -sSfL https://git.io/getwoke | bash -s -- -b ./; }

uv-lock-check: ## Check that the uv.lock file is in a good shape
	uv lock --check

install-deps: install-tools uv-lock-check ## Install all required dependencies needed to run the service, according to uv.lock
	@for a in 1 2 3 4 5; do uv sync && break || sleep 15; done

install-deps-test: install-tools uv-lock-check ## Install all required dev dependencies needed to test the service, according to uv.lock
	@for a in 1 2 3 4 5; do uv sync --group dev && break || sleep 15; done

update-deps: ## Upgrade ALL locked packages then sync. Do not use for a single CVE/package bump.
	uv lock --upgrade && uv sync

run: ## Run the service locally
	uv run python runner.py

print-version: ## Print the service version
	uv run python runner.py --version

test: test-unit test-integration test-e2e ## Run all tests

benchmarks: ## Run benchmarks
	@echo "Running benchmarks..."
	uv run pytest tests/benchmarks --benchmark-histogram

test-unit: ## Run the unit tests
	@echo "Running unit tests..."
	@echo "Reports will be written to ${ARTIFACT_DIR}"
	COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.unit" uv run pytest tests/unit --cov=ols --cov=runner --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_unit.json" --junit-xml="${ARTIFACT_DIR}/junit_unit.xml"
	uv run scripts/transform_coverage_report.py "${ARTIFACT_DIR}/coverage_unit.json" "${ARTIFACT_DIR}/coverage_unit.out"
	scripts/codecov.sh "${ARTIFACT_DIR}/coverage_unit.out"

test-integration: ## Run integration tests tests
	@echo "Running integration tests..."
	@echo "Reports will be written to ${ARTIFACT_DIR}"
	COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.integration" uv run pytest tests/integration --cov=ols --cov=runner --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_integration.json" --junit-xml="${ARTIFACT_DIR}/junit_integration.xml" --cov-fail-under=60
	uv run scripts/transform_coverage_report.py "${ARTIFACT_DIR}/coverage_integration.json" "${ARTIFACT_DIR}/coverage_integration.out"
	scripts/codecov.sh "${ARTIFACT_DIR}/coverage_integration.out"

check-coverage: test-unit test-integration  ## Unit tests and integration tests overall code coverage check
	uv run coverage combine --keep "${ARTIFACT_DIR}/.coverage.unit" "${ARTIFACT_DIR}/.coverage.integration"
	# the threshold should be very high there, in theory it should reach 100%
	uv run coverage report -m --fail-under=94

test-e2e: ## Run e2e tests - requires running OLS server
	@echo "Running e2e tests..."
	@echo "Reports will be written to ${ARTIFACT_DIR}"
	uv run pytest tests/e2e --ignore=tests/e2e/evaluation -s --durations=0 -o junit_suite_name="${SUITE_ID}" -m "${TEST_TAGS}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
	--eval_provider ${PROVIDER} --eval_model ${MODEL} --eval_out_dir ${ARTIFACT_DIR}

test-lseval-periodic: ## Run LSEval periodic evaluation (full 797-question dataset) - requires running OLS server with OpenAI keys
	@echo "Running LSEval periodic evaluation..."
	@echo "Reports will be written to ${ARTIFACT_DIR}"
	uv run --extra lseval --extra evaluation pytest tests/e2e/evaluation/test_lseval_periodic.py -vv -s --durations=0 -o junit_suite_name="${SUITE_ID}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
	--eval_out_dir ${ARTIFACT_DIR} -m lseval --lseval_provider ${PROVIDER}

test-lseval-presubmit: ## Run LSEval presubmit evaluation (10-question short dataset) - requires running OLS server
	@echo "Running LSEval presubmit evaluation..."
	@echo "Reports will be written to ${ARTIFACT_DIR}"
	uv run --extra lseval --extra evaluation pytest tests/e2e/evaluation/test_lseval_presubmit.py -vv -s --durations=0 -o junit_suite_name="${SUITE_ID}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
	--eval_out_dir ${ARTIFACT_DIR} -m lseval --lseval_provider ${PROVIDER}

# DISABLED: re-enable by uncommenting the recipe body below.
# test-lseval-troubleshooting: ## Run LSEval troubleshooting evaluation (scenario + MCP suites) - requires running OLS server with OpenAI keys
# 	@echo "Running LSEval troubleshooting evaluation..."
# 	@echo "Reports will be written to ${ARTIFACT_DIR}"
# 	uv run --extra lseval --extra evaluation pytest tests/e2e/evaluation/test_lseval_troubleshooting.py -vv -s --durations=0 -o junit_suite_name="${SUITE_ID}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
# 	--eval_out_dir ${ARTIFACT_DIR} -m lseval

test-lseval-troubleshooting: ## DISABLED: LSEval troubleshooting (uncomment Makefile to re-enable)
	@echo "test-lseval-troubleshooting is disabled for now (troubleshooting LSEval commented out)." >&2

test-cluster-updates: ## Run cluster-updates evaluation (18 conversations, 35 evaluations) - requires running OLS server with OpenAI keys
	@echo "Running cluster-updates evaluation..."
	@echo "Reports will be written to ${ARTIFACT_DIR}"
	uv run --extra lseval --extra evaluation pytest tests/e2e/evaluation -vv -s --durations=0 -o junit_suite_name="${SUITE_ID}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
	--eval_out_dir ${ARTIFACT_DIR} -m cluster_updates

coverage-report:	unit-tests-coverage-report integration-tests-coverage-report ## Export coverage reports into interactive HTML

unit-tests-coverage-report:	test-unit ## Export unit test coverage report into interactive HTML
	uv run coverage html --data-file="${ARTIFACT_DIR}/.coverage.unit" -d htmlcov-unit

integration-tests-coverage-report:	test-integration ## Export integration test coverage report into interactive HTML
	uv run coverage html --data-file="${ARTIFACT_DIR}/.coverage.integration" -d htmlcov-integration

check-types: ## Checks type hints in sources
	uv run mypy --explicit-package-bases --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs ols/

security-check: ## Check the project for security issues
	uv run bandit -c pyproject.toml -r ols runner.py

format: ## Format the code into unified format
	uv run black .
	uv run ruff check . --fix

verify:	install-woke install-deps-test verify-hermetic-requirements ## Verify the code using various linters
	uv run black . --check
	uv run ruff check .
	./woke . --exit-1-on-failure
	uv run --extra evaluation pylint ols scripts tests runner.py
	uv run mypy --explicit-package-bases --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs ols/

verify-hermetic-requirements:	## Verify hermetic build hash files are in sync with uv.lock
	bash scripts/verify_hermetic_requirements.sh

schema:	## Generate OpenAPI schema file
	uv run python scripts/generate_openapi_schema.py docs/openapi.json

requirements.txt:	pyproject.toml uv.lock ## Generate requirements.txt file containing hashes for all non-devel packages
	uv export --format requirements-txt --no-dev --no-extra evaluation --no-editable --no-emit-package ols --output-file requirements.txt

verify-packages-completeness:	requirements.txt ## Verify that requirements.txt file contains complete list of packages
	pip download -d /tmp/ --use-pep517 --verbose -r requirements.txt

get-embeddings: get-embeddings-byok get-embeddings-okp ## Download all embedding model binaries

get-embeddings-byok: ## Download and split BYOK/FAISS embedding model binary
	curl --fail --location --retry 3 --retry-delay 2 --connect-timeout 15 \
		-o embeddings_model/all-mpnet-base-v2/model.safetensors \
		"https://huggingface.co/sentence-transformers/all-mpnet-base-v2/resolve/main/model.safetensors"
	cd embeddings_model/all-mpnet-base-v2 && \
		rm -f model.safetensors.tar.gz.* && \
		tar czf - model.safetensors | split -b 95M - model.safetensors.tar.gz.

get-embeddings-okp: ## Download and split OKP/RAG embedding model binary
	curl --fail --location --retry 3 --retry-delay 2 --connect-timeout 15 \
		-o embeddings_model/granite-embedding-30m-english/model.safetensors \
		"https://huggingface.co/ibm-granite/granite-embedding-30m-english/resolve/main/model.safetensors"
	cd embeddings_model/granite-embedding-30m-english && \
		rm -f model.safetensors.tar.gz.* && \
		tar czf - model.safetensors | split -b 95M - model.safetensors.tar.gz.

config.puml: ## Generate PlantUML class diagram for configuration
	pyreverse ols/app/models/config.py --output puml --output-directory=docs/
	mv docs/classes.puml docs/config.puml

docs/config.png:	docs/config.puml ## Generate an image with configuration graph
	pushd docs && \
	java -jar ${PATH_TO_PLANTUML}/plantuml.jar --theme rose config.puml && \
	mv classes.png config.png && \
	popd

llms.puml: ## Generate PlantUML class diagram for LLM plugin system
	pyreverse ols/src/llms/ --output puml --output-directory=docs/
	mv docs/classes.puml docs/llms_classes.uml
	mv docs/packages.puml docs/llms_packages.uml

distribution-archives: ## Generate distribution archives to be uploaded into Python registry
	uv run python -m build

upload-distribution-archives: ## Upload distribution archives into Python registry
	uv run python -m twine upload --repository ${PYTHON_REGISTRY} dist/*

shellcheck: ## Run shellcheck
	wget -qO- "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" | tar -xJv \
	shellcheck --version
	shellcheck -- */*.sh

tls-scan: ## Run TLS profile compliance scan against OLS endpoints
	ARTIFACT_DIR="$(ARTIFACT_DIR)" ./scripts/tls-scan.sh

konflux-requirements:	## Generate hermetic requirements.*.txt file for konflux build
	python3 scripts/konflux_resolve.py --profile cpu

konflux-rpm-lock:	## Generate rpm.lock.yaml file for konflux build
	./scripts/generate-rpm-lock.sh -a ${ACTIVATION_KEY} -g ${ORG_ID}

help: ## Show this help screen
	@echo 'Usage: make <OPTIONS> ... <TARGETS>'
	@echo ''
	@echo 'Available targets are:'
	@echo ''
	@grep -E '^[ a-zA-Z0-9_.-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-33s\033[0m %s\n", $$1, $$2}'
	@echo ''