diff --git a/backend/requirements.txt b/backend/requirements.txt index 0570f52..5b484a9 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,3 +1,4 @@ +python-dotenv>=1.0.1 PyJWT>=2.8.0 fastapi[standard]>=0.116.1 pydantic>=2.11.7 diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 6ac2dab..f55bcf8 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,810 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail - -# Complete deployment script: build and push to server -# Usage: ./scripts/deploy.sh [server_user@server_host] [deployment_path] [platform] -# Example: ./scripts/deploy.sh user@example.com /home/user/fromchat linux/arm64 - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -CYAN='\033[0;36m' -MAGENTA='\033[0;35m' -NC='\033[0m' # No Color -BOLD='\033[1m' - -# Helper functions -info() { echo -e "${BLUE}ℹ${NC} $1"; } -success() { echo -e "${GREEN}✓${NC} $1"; } -warning() { echo -e "${YELLOW}⚠${NC} $1"; } -error() { echo -e "${RED}✗${NC} $1"; } -step() { echo -e "${CYAN}${BOLD}→${NC} ${BOLD}$1${NC}"; } -substep() { - if [ "${2:-}" = "-n" ]; then - echo -n -e " ${GREEN}•${NC} $1" - else - echo -e " ${GREEN}•${NC} $1" - fi -} - -echo -e "${MAGENTA}${BOLD}🚀 Deployment${NC}\n" - - -read_password() { - local password="" - local char - local old_stty - - old_stty=$(stty -g 2>/dev/null) - stty -echo 2>/dev/null - - while IFS= read -rs -n 1 char; do - if [ -z "$char" ]; then - break - fi - if [ "$char" = $'\177' ] || [ "$char" = $'\b' ]; then - if [ ${#password} -gt 0 ]; then - password="${password%?}" - printf "\b \b" >&2 - fi - else - password+="$char" - printf "*" >&2 - fi - done - - stty "$old_stty" 2>/dev/null - echo "" >&2 - echo "$password" -} - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -DEPLOYMENT_DIR="$PROJECT_ROOT/deployment" -ENV_FILE="$DEPLOYMENT_DIR/.env" -INPUT_HASH_TOOL="$SCRIPT_DIR/docker_inputs_hash.py" -LOCAL_CACHE_ROOT="$PROJECT_ROOT/.deploy-cache" -LOCAL_IMAGE_CACHE_DIR="$LOCAL_CACHE_ROOT/images" - -# Ensure arrays exist even when build is skipped (nounset-safe). -declare -a BUILT_IMAGES - -# Load .env file if it exists -if [ -f "$ENV_FILE" ]; then - # Export variables from .env file (ignore comments and empty lines) - set -a - while IFS= read -r line || [ -n "$line" ]; do - # Skip comments and empty lines - case "$line" in - \#*|'') continue ;; - *) - # Export the variable - export "$line" 2>/dev/null || true - ;; - esac - done < "$ENV_FILE" - set +a -fi - -# Read server from environment variable (from .env), command line argument, or fallback -SERVER="${1:-${DEPLOYMENT_SERVER:-}}" -REPO_NAME="FromChat" -DEPLOY_PATH="~/actions-runner/_work/$REPO_NAME/$REPO_NAME" -PLATFORM="linux/arm64" - -# Prefer plain `docker build` when target arch matches host arch. -# This avoids Docker Desktop buildx export/load issues and is faster for same-arch builds. -HOST_ARCH_RAW="$(uname -m 2>/dev/null || echo "")" -case "$HOST_ARCH_RAW" in - arm64|aarch64) HOST_ARCH="arm64" ;; - x86_64|amd64) HOST_ARCH="amd64" ;; - *) HOST_ARCH="$HOST_ARCH_RAW" ;; -esac -PLATFORM_ARCH="${PLATFORM##*/}" -USE_DOCKER_BUILD=false -if [ -n "$HOST_ARCH" ] && [ "$HOST_ARCH" = "$PLATFORM_ARCH" ]; then - USE_DOCKER_BUILD=true -fi - -sanitize_ref() { - # Replace characters that are problematic for file paths. - # Example: "foo/bar:baz" -> "foo_bar__baz" - local s="$1" - s="${s//\//_}" - s="${s//:/__}" - s="${s//@/__at__}" - echo "$s" -} - -read_file_if_exists() { - local p="$1" - if [ -f "$p" ]; then - cat "$p" - fi -} - -# Docker's top-level image Id can differ for the same layers (OCI index vs image, BuildKit attestations). -# Compare a fingerprint of the final layer stack instead. -get_local_image_layer_fp() { - local img="$1" - docker image inspect -f '{{json .RootFS.Layers}}' "$img" 2>/dev/null \ - | openssl dgst -sha256 2>/dev/null | awk '{print $2}' || true -} - -compute_inputs_hash() { - local context="$1" - local dockerfile="$2" - python3 "$INPUT_HASH_TOOL" --context "$context" --dockerfile "$dockerfile" -} - -# Check if server is provided -if [ -z "$SERVER" ]; then - error "Server not specified. Usage: $0 [user@host] [deployment_path] [platform]" - echo " Or set DEPLOYMENT_SERVER in $ENV_FILE or as an environment variable" - echo "" - echo "Example:" - echo " $0 user@example.com /home/user/fromchat linux/arm64" - echo " Or add to $ENV_FILE: DEPLOYMENT_SERVER=user@example.com" - echo " Or: DEPLOYMENT_SERVER=user@example.com $0" - exit 1 -fi - -# ============================================================================ -# SSH AUTHENTICATION -# ============================================================================ - -step "Authentication" -SSH_KEY_FILE="$HOME/.ssh/id_rsa" -SSH_KEY_PUB_FILE="$SSH_KEY_FILE.pub" - -# Ensure ssh-agent is running -if [ -z "$SSH_AUTH_SOCK" ]; then - eval "$(ssh-agent -s)" > /dev/null 2>&1 -fi - -# Check if SSH key exists -if [ ! -f "$SSH_KEY_FILE" ]; then - error "SSH key not found at $SSH_KEY_FILE" - echo " Please generate an SSH key pair first:" - echo " ssh-keygen -t rsa -b 4096 -C 'your_email@example.com'" - exit 1 -fi - -# Add SSH key to agent if not already loaded -KEY_LOADED=false -if ssh-add -l > /dev/null 2>&1; then - # Check if this specific key is loaded by trying to match the public key - KEY_FINGERPRINT=$(ssh-keygen -lf "$SSH_KEY_FILE" 2>/dev/null | awk '{print $2}') - if [ -n "$KEY_FINGERPRINT" ] && ssh-add -l 2>/dev/null | grep -q "$KEY_FINGERPRINT"; then - KEY_LOADED=true - fi -fi - -if [ "$KEY_LOADED" = false ]; then - substep "Adding SSH key to agent..." - if ! ssh-add "$SSH_KEY_FILE" 2>/dev/null; then - error "Failed to add SSH key to agent. Check your key passphrase." - exit 1 - fi -fi - -# Require key-based SSH auth; do not attempt to copy keys automatically. -if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SERVER" "echo 'SSH key works'" >/dev/null 2>&1; then - error "SSH key authentication failed for $SERVER" - echo " Copy your public key to the server, then re-run deploy:" - echo " ssh-copy-id -i \"$SSH_KEY_PUB_FILE\" \"$SERVER\"" - echo "" - echo " Or manually append this key to ~/.ssh/authorized_keys on the server:" - echo " $(cat "$SSH_KEY_PUB_FILE")" - exit 1 -fi - -# ============================================================================ -# SUDO AUTHENTICATION -# ============================================================================ - -SUDO_PASSWORD="" -# Prompt for sudo password (optional; leave blank for passwordless sudo) -if [ -z "$SUDO_PASSWORD" ]; then - while true; do - substep "Sudo password: " -n - SUDO_PASSWORD=$(read_password) - - if [ -z "$SUDO_PASSWORD" ]; then - warning "No password provided - assuming passwordless sudo" - break - fi - - if echo "$SUDO_PASSWORD" | ssh "$SERVER" "sudo -S -v" > /dev/null 2>&1; then - export SUDO_PASSWORD - break - else - echo -n " " && error "Invalid password, please try again" - fi - done -fi - -# ============================================================================ -# BUILD PHASE -# ============================================================================ - -echo -e "\n${MAGENTA}${BOLD}🔨 Building Docker images${NC}\n" - -# Determine project name -if [ -n "$SERVER" ]; then - COMPOSE_DIR=$(ssh "$SERVER" "dirname $DEPLOY_PATH/deployment/docker-compose.yml" 2>/dev/null || echo "$DEPLOY_PATH/deployment") - PROJECT_NAME=$(ssh "$SERVER" "basename $COMPOSE_DIR" 2>/dev/null || echo "deployment") -else - PROJECT_NAME=$(basename "$DEPLOYMENT_DIR") -fi - -# Check if Docker daemon is running -check_docker_daemon() { - docker info > /dev/null 2>&1 -} - -# Start Docker Desktop -start_docker_desktop() { - substep "Starting Docker Desktop..." - if ! docker desktop start > /dev/null 2>&1; then - return 1 - fi - - # Wait for Docker to be ready (max 60 seconds) - substep "Waiting for Docker to start..." -n - local max_wait=60 - local waited=0 - while [ $waited -lt $max_wait ]; do - if check_docker_daemon; then - echo "" - return 0 - fi - sleep 2 - waited=$((waited + 2)) - echo -n "." - done - echo "" - return 1 -} - -# Check Docker daemon -if ! check_docker_daemon; then - warning "Docker daemon is not running" - if ! start_docker_desktop; then - error "Failed to start Docker Desktop. Please start it manually and try again." - fi -fi - -if [ "$USE_DOCKER_BUILD" = false ]; then - # Check buildx - if ! docker buildx version > /dev/null 2>&1; then - error "Docker buildx not available. Install Docker Desktop." - fi - - # Setup buildx builder - step "Setting up buildx builder" - BUILDER_NAME="fromchat-builder" - BUILDER_EXISTS=false - - if docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then - BUILDER_EXISTS=true - if ! docker buildx use "$BUILDER_NAME" > /dev/null 2>&1; then - substep "Recreating builder..." - docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true - BUILDER_EXISTS=false - elif ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then - substep "Recreating builder (inspection failed)..." - docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true - BUILDER_EXISTS=false - fi - fi - - if [ "$BUILDER_EXISTS" = false ]; then - substep "Creating builder with persistent cache..." - docker buildx create \ - --name "$BUILDER_NAME" \ - --driver docker-container \ - --driver-opt image=moby/buildkit:latest \ - --use \ - --bootstrap > /dev/null 2>&1 - fi - - docker buildx use "$BUILDER_NAME" > /dev/null 2>&1 -fi - -# Detect services -step "Detecting services" -cd "$DEPLOYMENT_DIR" -# Include profile-only services (e.g. caddy) so their images are built and pushed; otherwise prod keeps a stale Caddy image and ignores Caddyfile updates from rsync. -export COMPOSE_PROFILES=production -SERVICES=$(docker compose -f docker-compose.yml config --services 2>/dev/null) - -if [ -z "$SERVICES" ]; then - error "No services found in docker-compose.yml" -fi - -if ! command -v jq >/dev/null 2>&1; then - error "jq is required for deploy (e.g. brew install jq / sudo apt install jq)" - exit 1 -fi - -if ! COMPOSE_JSON=$(docker compose -f docker-compose.yml config --format json 2>/dev/null); then - error "docker compose config --format json failed (needs Docker Compose v2.10+)" - exit 1 -fi - -BUILT_IMAGES=() -PUSHABLE_IMAGES=() -PUSHABLE_SERVICES=() -PUSHABLE_SERVICE_DOCKERFILES=() -PUSHABLE_SERVICE_CONTEXTS=() -PUSHABLE_SERVICE_TARGETS=() -PUSHABLE_IMAGE_INPUT_HASHES=() - -for SERVICE in $SERVICES; do - if ! jq -e --arg s "$SERVICE" '(.services[$s].build // false) | type == "object"' <<< "$COMPOSE_JSON" >/dev/null 2>&1; then - continue - fi - - IMAGE_TAG="${PROJECT_NAME}-${SERVICE}:latest" - - DOCKERFILE_REL=$(jq -r --arg s "$SERVICE" '.services[$s].build.dockerfile // empty' <<< "$COMPOSE_JSON") - CONTEXT_REL=$(jq -r --arg s "$SERVICE" '.services[$s].build.context // empty' <<< "$COMPOSE_JSON") - # Multi-stage deployment/Dockerfile: without --target, the final stage (file_storage) is always tagged. - BUILD_TARGET=$(jq -r --arg s "$SERVICE" '.services[$s].build.target // empty' <<< "$COMPOSE_JSON") - - if [ -z "$CONTEXT_REL" ]; then - CONTEXT_REL=".." - fi - - if [[ "$CONTEXT_REL" == ".." ]]; then - BUILD_CONTEXT="$PROJECT_ROOT" - elif [[ "$CONTEXT_REL" == /* ]]; then - BUILD_CONTEXT="$CONTEXT_REL" - else - BUILD_CONTEXT="$DEPLOYMENT_DIR/$CONTEXT_REL" - fi - - if [ -n "$DOCKERFILE_REL" ]; then - if [[ "$DOCKERFILE_REL" == /* ]]; then - DOCKERFILE="$DOCKERFILE_REL" - else - if [[ "$CONTEXT_REL" == ".." ]] || [[ "$BUILD_CONTEXT" == "$PROJECT_ROOT" ]]; then - DOCKERFILE="$PROJECT_ROOT/$DOCKERFILE_REL" - else - DOCKERFILE="$BUILD_CONTEXT/$DOCKERFILE_REL" - fi - fi - else - if [ -f "$DEPLOYMENT_DIR/Dockerfile.$SERVICE" ]; then - DOCKERFILE="$DEPLOYMENT_DIR/Dockerfile.$SERVICE" - elif [ -f "$DEPLOYMENT_DIR/$SERVICE/Dockerfile" ]; then - DOCKERFILE="$DEPLOYMENT_DIR/$SERVICE/Dockerfile" - else - error "Could not determine Dockerfile for $SERVICE" - fi - fi - - if [ ! -f "$INPUT_HASH_TOOL" ]; then - error "Missing $INPUT_HASH_TOOL (needed for dependency hashing)" - exit 1 - fi - - INPUT_HASH=$(compute_inputs_hash "$BUILD_CONTEXT" "$DOCKERFILE") - - PUSHABLE_SERVICES+=("$SERVICE") - PUSHABLE_IMAGES+=("$IMAGE_TAG") - PUSHABLE_SERVICE_DOCKERFILES+=("$DOCKERFILE") - PUSHABLE_SERVICE_CONTEXTS+=("$BUILD_CONTEXT") - PUSHABLE_SERVICE_TARGETS+=("$BUILD_TARGET") - PUSHABLE_IMAGE_INPUT_HASHES+=("$INPUT_HASH") -done - -mkdir -p "$LOCAL_IMAGE_CACHE_DIR" - -SERVICES_TO_BUILD=() -IMAGES_TO_BUILD=() -INPUT_HASHES_TO_BUILD=() -DOCKERFILES_TO_BUILD=() -CONTEXTS_TO_BUILD=() -TARGETS_TO_BUILD=() - -for i in "${!PUSHABLE_SERVICES[@]}"; do - SERVICE="${PUSHABLE_SERVICES[$i]}" - IMAGE_TAG="${PUSHABLE_IMAGES[$i]}" - INPUT_HASH="${PUSHABLE_IMAGE_INPUT_HASHES[$i]}" - DOCKERFILE="${PUSHABLE_SERVICE_DOCKERFILES[$i]}" - BUILD_CONTEXT="${PUSHABLE_SERVICE_CONTEXTS[$i]}" - BUILD_TARGET="${PUSHABLE_SERVICE_TARGETS[$i]}" - - CACHE_KEY="$(sanitize_ref "$IMAGE_TAG")" - CACHE_FILE="$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY/input.sha256" - PREV_HASH="$(read_file_if_exists "$CACHE_FILE")" - LOCAL_FP="$(get_local_image_layer_fp "$IMAGE_TAG")" - - if [ -n "$PREV_HASH" ] && [ "$PREV_HASH" = "$INPUT_HASH" ] && [ -n "$LOCAL_FP" ]; then - continue - fi - - SERVICES_TO_BUILD+=("$SERVICE") - IMAGES_TO_BUILD+=("$IMAGE_TAG") - INPUT_HASHES_TO_BUILD+=("$INPUT_HASH") - DOCKERFILES_TO_BUILD+=("$DOCKERFILE") - CONTEXTS_TO_BUILD+=("$BUILD_CONTEXT") - TARGETS_TO_BUILD+=("$BUILD_TARGET") -done - -if [ ${#SERVICES_TO_BUILD[@]} -eq 0 ]; then - success "Build skipped (no Docker inputs changed)" -else - step "Building ${#SERVICES_TO_BUILD[@]} service(s)" - - if [ "$USE_DOCKER_BUILD" = true ]; then - # Same-arch fast path: build with Docker Compose (requested). - # Ensure the project name is stable so the resulting tags match our expectations. - COMPOSE_PROJECT_NAME="$PROJECT_NAME" docker compose -f docker-compose.yml --profile production build "${SERVICES_TO_BUILD[@]}" - else - for i in "${!SERVICES_TO_BUILD[@]}"; do - SERVICE="${SERVICES_TO_BUILD[$i]}" - IMAGE_TAG="${IMAGES_TO_BUILD[$i]}" - DOCKERFILE="${DOCKERFILES_TO_BUILD[$i]}" - BUILD_CONTEXT="${CONTEXTS_TO_BUILD[$i]}" - BUILD_TARGET="${TARGETS_TO_BUILD[$i]}" - - substep "Building ${CYAN}$SERVICE${NC} -> ${CYAN}$IMAGE_TAG${NC}..." - - # On macOS Docker Desktop, --load can hang for a long time at "sending tarball". - # Use the docker exporter explicitly to load into the local Docker daemon. - BUILDX_ARGS=(buildx build --platform "$PLATFORM" --file "$DOCKERFILE" --tag "$IMAGE_TAG" --output=type=docker --provenance=false --sbom=false) - if [ -n "$BUILD_TARGET" ]; then - BUILDX_ARGS+=(--target "$BUILD_TARGET") - fi - BUILDX_ARGS+=("$BUILD_CONTEXT") - - if ! docker "${BUILDX_ARGS[@]}"; then - error "Build failed for $SERVICE" - exit 1 - fi - done - fi - - for i in "${!SERVICES_TO_BUILD[@]}"; do - IMAGE_TAG="${IMAGES_TO_BUILD[$i]}" - INPUT_HASH="${INPUT_HASHES_TO_BUILD[$i]}" - CACHE_KEY="$(sanitize_ref "$IMAGE_TAG")" - mkdir -p "$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY" - printf "%s" "$INPUT_HASH" > "$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY/input.sha256" - BUILT_IMAGES+=("$IMAGE_TAG") - done - - success "Build complete! ${#BUILT_IMAGES[@]} image(s) built" -fi - -# ============================================================================ -# DEPLOY PHASE -# ============================================================================ - -echo -e "\n${MAGENTA}${BOLD}🚀 Deploying to ${SERVER}${NC}\n" - - -# Check docker pussh -if ! docker pussh --help > /dev/null 2>&1; then - error "docker pussh plugin not installed" - echo " Install: npm run install:pussh" -fi - -# Detect images based on docker-compose.yml (prefer explicit `image:` entries; fall back to built tags) -cd "$DEPLOYMENT_DIR" -COMPOSE_SERVICES=$(docker compose -f docker-compose.yml config --services 2>/dev/null || true) -PUSH_IMAGES=() -EXTERNAL_IMAGES=() - -for S in $COMPOSE_SERVICES; do - IMAGE_FROM_COMPOSE=$(jq -r --arg s "$S" '.services[$s].image // empty' <<< "$COMPOSE_JSON") - HAS_BUILD=$(jq -r --arg s "$S" '(.services[$s].build // empty) | (if . == "" then "" else "yes" end)' <<< "$COMPOSE_JSON") - - if [ -n "$IMAGE_FROM_COMPOSE" ]; then - # If it has an explicit image and no build section, it likely won't exist locally (and doesn't need pussh). - if [ -z "$HAS_BUILD" ]; then - EXTERNAL_IMAGES+=("$IMAGE_FROM_COMPOSE") - else - # If a service has both build+image, treat it as a built image (pussh). - PUSH_IMAGES+=("$IMAGE_FROM_COMPOSE") - fi - else - # If service has a build section (we built it above), use the tag pattern used during build - TAG="${PROJECT_NAME}-${S}:latest" - # Only include the tag if the image exists locally (avoid pushing unrelated images) - if docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "^${TAG}$"; then - PUSH_IMAGES+=("$TAG") - fi - fi -done - -# Deduplicate while preserving order -if [ ${#PUSH_IMAGES[@]} -gt 0 ]; then - PUSH_IMAGES=($(printf "%s\n" "${PUSH_IMAGES[@]}" | awk '!seen[$0]++')) -fi -if [ ${#EXTERNAL_IMAGES[@]} -gt 0 ]; then - EXTERNAL_IMAGES=($(printf "%s\n" "${EXTERNAL_IMAGES[@]}" | awk '!seen[$0]++')) -fi - -# Verify that all built images are among the detected images to be pushed. -# This prevents accidentally pushing unrelated images. -BUILT_COUNT=${#BUILT_IMAGES[@]} -MATCHING_BUILT=0 -MISSING_FROM_DETECTED=() -# Bash 3.2 + set -u: "${arr[@]}" on an empty array can error; guard with length. -if ((${#BUILT_IMAGES[@]} > 0)); then - for BI in "${BUILT_IMAGES[@]}"; do - found=false - for DI in "${PUSH_IMAGES[@]}"; do - if [ "$BI" = "$DI" ]; then - found=true - break - fi - done - if [ "$found" = true ]; then - MATCHING_BUILT=$((MATCHING_BUILT + 1)) - else - MISSING_FROM_DETECTED+=("$BI") - fi - done -fi - -# Also list push images that weren't built locally (these are likely prebuilt local images) -NOT_BUILT_DETECTED=() -for DI in "${PUSH_IMAGES[@]}"; do - built=false - if ((${#BUILT_IMAGES[@]} > 0)); then - for BI in "${BUILT_IMAGES[@]}"; do - if [ "$DI" = "$BI" ]; then - built=true - break - fi - done - fi - if [ "$built" = false ]; then - NOT_BUILT_DETECTED+=("$DI") - fi -done - -if [ "$BUILT_COUNT" -ne "$MATCHING_BUILT" ]; then - error "Mismatch between built images (${BUILT_COUNT}) and detected built images (${MATCHING_BUILT})." - if [ ${#MISSING_FROM_DETECTED[@]} -gt 0 ]; then - echo " Built but not detected: ${MISSING_FROM_DETECTED[*]}" - fi - if [ ${#NOT_BUILT_DETECTED[@]} -gt 0 ]; then - echo " Detected but not built (external images): ${NOT_BUILT_DETECTED[*]}" - fi - echo "Aborting to avoid pushing incorrect images." - exit 1 -fi - -if [ ${#PUSH_IMAGES[@]} -eq 0 ] && [ ${#EXTERNAL_IMAGES[@]} -eq 0 ]; then - error "No images found in docker-compose.yml or built locally for project ${PROJECT_NAME}" -fi - -# Transfer images — only what was rebuilt this run (BUILT_IMAGES). No remote hash/fingerprint checks. -step "Transferring images" - -IMAGES_TO_PUSH=() -if ((${#BUILT_IMAGES[@]} > 0)); then - for PI in "${PUSH_IMAGES[@]}"; do - for BI in "${BUILT_IMAGES[@]}"; do - if [ "$PI" = "$BI" ]; then - IMAGES_TO_PUSH+=("$PI") - break - fi - done - done -fi - -if [ ${#IMAGES_TO_PUSH[@]} -eq 0 ]; then - success "Skipping image push (nothing was rebuilt this run)" -else - UNREGISTRY_IMAGE="ghcr.io/psviderski/unregistry" - if ! ssh "$SERVER" "sudo docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^${UNREGISTRY_IMAGE}$'" 2>/dev/null; then - substep "Pulling unregistry image (one-time setup)..." - if [ -n "${SUDO_PASSWORD:-}" ]; then - ssh "$SERVER" "echo '$SUDO_PASSWORD' | sudo -S -p '' docker pull ${UNREGISTRY_IMAGE}" - else - ssh "$SERVER" "sudo docker pull ${UNREGISTRY_IMAGE}" - fi - fi - for IMAGE in "${IMAGES_TO_PUSH[@]}"; do - substep "Pushing ${CYAN}$IMAGE${NC}..." - if ! docker pussh "$IMAGE" "$SERVER"; then - echo -e " ${RED}✗${NC} Failed to push ${CYAN}$IMAGE${NC}" - echo "" - exit 1 - fi - echo "" - done -fi - -# Pull external images directly on the server (no pussh) -if [ ${#EXTERNAL_IMAGES[@]} -gt 0 ]; then - step "Pulling external images on server" - for IMAGE in "${EXTERNAL_IMAGES[@]}"; do - substep "Pulling ${CYAN}$IMAGE${NC}..." - QIMG=$(printf '%q' "$IMAGE") - if [ -n "${SUDO_PASSWORD:-}" ]; then - if ssh "$SERVER" "echo '$SUDO_PASSWORD' | sudo -S -p '' docker pull $QIMG"; then - echo "" - else - echo -e " ${RED}✗${NC} Failed to pull ${CYAN}$IMAGE${NC} on server" - echo "" - exit 1 - fi - elif ssh "$SERVER" "sudo docker pull $QIMG"; then - echo "" - else - echo -e " ${RED}✗${NC} Failed to pull ${CYAN}$IMAGE${NC} on server" - echo "" - exit 1 - fi - done -fi - -# Transfer files -step "Transferring deployment files" - -# Ensure destination directory exists with proper permissions -if [ -n "$SUDO_PASSWORD" ]; then - ssh "$SERVER" bash << REMOTE_SUDO_SCRIPT > /dev/null 2>&1 -set -e -echo '$SUDO_PASSWORD' | sudo -S -p '' mkdir -p $DEPLOY_PATH/deployment $DEPLOY_PATH/backend 2>/dev/null || true -echo '$SUDO_PASSWORD' | sudo -S -p '' chown -R \$(whoami):\$(whoami) $DEPLOY_PATH/deployment $DEPLOY_PATH/backend 2>/dev/null || true -REMOTE_SUDO_SCRIPT -else - ssh "$SERVER" "sudo mkdir -p $DEPLOY_PATH/deployment $DEPLOY_PATH/backend && sudo chown -R \$(whoami):\$(whoami) $DEPLOY_PATH/deployment $DEPLOY_PATH/backend" > /dev/null 2>&1 || true -fi - -# Copy deployment directory excluding gitignored files -cd "$PROJECT_ROOT" -substep "Copying deployment directory..." - -# Generate exclude file for rsync using git ls-files to list ignored files -EXCLUDE_FILE="/tmp/fromchat-rsync-exclude-$$" -RSYNC_ERROR="/tmp/fromchat-rsync-error-$$" - -# Get ignored files in deployment directory and convert to rsync exclude patterns -git ls-files --others --ignored --exclude-standard deployment/ 2>/dev/null | \ - sed 's|^deployment/||' > "$EXCLUDE_FILE" || true - -# Use rsync with native --exclude-from option -if rsync -avz --delete --exclude-from="$EXCLUDE_FILE" \ - "$DEPLOYMENT_DIR/" \ - "$SERVER:$DEPLOY_PATH/deployment/" > "$RSYNC_ERROR" 2>&1; then - rm -f "$EXCLUDE_FILE" "$RSYNC_ERROR" -else - echo -e " ${RED}✗${NC} Rsync failed. Error output:" - cat "$RSYNC_ERROR" | sed 's/^/ /' - rm -f "$EXCLUDE_FILE" "$RSYNC_ERROR" - echo -n " " && error "Failed to copy deployment directory" -fi - -# Copy .env.prod to .env on server (bypassing gitignore) -if [ -f "$DEPLOYMENT_DIR/.env.prod" ]; then - substep "Copying .env.prod to .env..." - if ! scp "$DEPLOYMENT_DIR/.env.prod" "$SERVER:$DEPLOY_PATH/deployment/.env" > /dev/null 2>&1; then - warning "Failed to copy .env.prod to .env" - fi -else - warning ".env.prod not found in deployment directory" -fi - -# Firebase service account: bind-mounted at runtime (see docker-compose main volumes), never in the image (.dockerignore). -# If compose ever ran without this file on the host, Docker may have created a directory at this path — remove it before scp. -FIREBASE_CERT="$PROJECT_ROOT/backend/firebase-cert.json" -substep "Firebase service account (runtime bind-mount: backend/firebase-cert.json)..." -# ~ is not expanded inside variables on the remote shell (e.g. C="$D/..." with D=~/foo checks a bogus path). Resolve to an absolute path. -DEPLOY_PATH_ON_SERVER=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$SERVER" "eval echo $DEPLOY_PATH" 2>/dev/null || true) -if [ -z "$DEPLOY_PATH_ON_SERVER" ]; then - DEPLOY_PATH_ON_SERVER=$DEPLOY_PATH -fi -FIREBASE_REMOTE="$DEPLOY_PATH_ON_SERVER/backend/firebase-cert.json" -# Docker may have created this path as a root-owned directory; plain rm fails without sudo. -if [ -n "$SUDO_PASSWORD" ]; then - ssh "$SERVER" bash << REMOTE_FIREBASE_CLEANUP > /dev/null 2>&1 || true -set -e -D=$DEPLOY_PATH_ON_SERVER -C="\$D/backend/firebase-cert.json" -mkdir -p "\$D/backend" 2>/dev/null || true -if [ -d "\$C" ]; then - echo '$SUDO_PASSWORD' | sudo -S -p '' rm -rf "\$C" -fi -echo '$SUDO_PASSWORD' | sudo -S -p '' chown -R "\$(whoami):\$(whoami)" "\$D/backend" 2>/dev/null || true -REMOTE_FIREBASE_CLEANUP -else - QBASE=$(printf '%q' "$DEPLOY_PATH_ON_SERVER") - ssh "$SERVER" "D=$QBASE; C=\"\$D/backend/firebase-cert.json\"; mkdir -p \"\$D/backend\"; if [ -d \"\$C\" ]; then sudo rm -rf \"\$C\" 2>/dev/null || rm -rf \"\$C\"; fi; sudo chown -R \$(whoami):\$(whoami) \"\$D/backend\" 2>/dev/null || true" > /dev/null 2>&1 || true -fi - -while true; do - if [ -f "$FIREBASE_CERT" ]; then - break - fi - if [ -d "$FIREBASE_CERT" ]; then - echo -e " ${YELLOW}⚠${NC} $FIREBASE_CERT is a directory. Delete it and save the Firebase service account JSON as a file at that exact path." - elif [ -e "$FIREBASE_CERT" ]; then - echo -e " ${YELLOW}⚠${NC} $FIREBASE_CERT exists but is not a regular file." - else - echo -e " ${YELLOW}⚠${NC} Missing $FIREBASE_CERT (Firebase service account JSON for FCM)." - fi - echo -e " ${CYAN}Fix this, then press Enter to check again (Ctrl+C to abort deploy).${NC}" - read -r _ -done - -substep "Copying backend/firebase-cert.json..." -FIREBASE_SCP_LOG="/tmp/fromchat-firebase-scp-$$.log" -if ! scp "$FIREBASE_CERT" "$SERVER:$FIREBASE_REMOTE" >"$FIREBASE_SCP_LOG" 2>&1; then - error "Failed to copy firebase-cert.json to server" - echo -e " ${YELLOW}Target:${NC} $SERVER:$FIREBASE_REMOTE" >&2 - if [ -s "$FIREBASE_SCP_LOG" ]; then - sed 's/^/ /' "$FIREBASE_SCP_LOG" >&2 - else - echo -e " ${YELLOW}(scp produced no output.)${NC}" >&2 - fi - rm -f "$FIREBASE_SCP_LOG" - exit 1 -fi -rm -f "$FIREBASE_SCP_LOG" -ssh "$SERVER" "chmod 600 $(printf '%q' "$FIREBASE_REMOTE") 2>/dev/null || true" -if ! ssh "$SERVER" "test -f $(printf '%q' "$FIREBASE_REMOTE")"; then - error "Server path is not a regular file after copy: $FIREBASE_REMOTE" - exit 1 -fi - -# Deploy on server -step "Deploying on server" -ssh "$SERVER" SUDO_PASSWORD="$SUDO_PASSWORD" DEPLOY_PATH="${DEPLOY_PATH_ON_SERVER:-$DEPLOY_PATH}" bash << 'REMOTE_SCRIPT' -set -e - -REMOTE_SUDO_PASS="${SUDO_PASSWORD:-}" -REMOTE_DEPLOY_PATH="${DEPLOY_PATH:-}" -export SUDO_PROMPT="" - -sudo_cmd() { - if [ -n "$REMOTE_SUDO_PASS" ]; then - echo "$REMOTE_SUDO_PASS" | sudo -S -p '' "$@" 2>/dev/null - else - sudo "$@" 2>/dev/null - fi -} - -if [ -z "$REMOTE_DEPLOY_PATH" ]; then - echo "❌ DEPLOY_PATH is not set" - exit 1 -fi - -mkdir -p "$REMOTE_DEPLOY_PATH/deployment" "$REMOTE_DEPLOY_PATH/backend" -cd "$REMOTE_DEPLOY_PATH/deployment" - -if [ ! -f "$REMOTE_DEPLOY_PATH/deployment/.env" ]; then - echo "⚠️ Warning: .env file not found" -fi - -if systemctl is-active --quiet fromchat; then - sudo_cmd systemctl stop fromchat -fi - -COMPOSE_PROFILES=production docker compose down --remove-orphans > /dev/null 2>&1 || true - -sudo_cmd cp -f "$REMOTE_DEPLOY_PATH/deployment/fromchat.service" /etc/systemd/system/fromchat.service -sudo_cmd systemctl daemon-reload -sudo_cmd systemctl restart fromchat - -sleep 3 -if ! systemctl is-active --quiet fromchat; then - echo "❌ Service failed to start" - sudo_cmd journalctl --no-pager -xeu fromchat -n 30 - exit 1 -fi -REMOTE_SCRIPT - -echo -success "Deployment complete!" +cd "$(dirname "$0")" +exec ../.venv/bin/python3 deploy/main.py "$@" diff --git a/scripts/deploy/__init__.py b/scripts/deploy/__init__.py new file mode 100644 index 0000000..7783d98 --- /dev/null +++ b/scripts/deploy/__init__.py @@ -0,0 +1 @@ +"""FromChat deployment orchestration (Docker build, pussh, rsync, remote systemd).""" diff --git a/scripts/deploy/compose_build.py b/scripts/deploy/compose_build.py new file mode 100644 index 0000000..eca6b84 --- /dev/null +++ b/scripts/deploy/compose_build.py @@ -0,0 +1,272 @@ +"""Parse docker-compose JSON and run image builds.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +from deploy.paths import ProjectPaths +import deploy.ui as ui +from deploy.util import ( + compute_inputs_hash, + dedupe_preserve, + local_image_layer_fp, + read_file_if_exists, + sanitize_ref, +) + + +def remote_project_name(server: str, deploy_path: str) -> str: + r = subprocess.run( + ["ssh", server, f"dirname {deploy_path}/deployment/docker-compose.yml"], + capture_output=True, + text=True, + ) + compose_dir = r.stdout.strip() if r.returncode == 0 and r.stdout.strip() else f"{deploy_path}/deployment" + r2 = subprocess.run(["ssh", server, f"basename {compose_dir}"], capture_output=True, text=True) + if r2.returncode == 0 and r2.stdout.strip(): + return r2.stdout.strip() + return "deployment" + + +@dataclass +class PushableService: + service: str + image_tag: str + dockerfile: Path + build_context: Path + build_target: str + input_hash: str + + +class ComposeBuildPhase: + def __init__( + self, + paths: ProjectPaths, + *, + project_name: str, + platform: str, + use_docker_build: bool, + ) -> None: + self._paths = paths + self._project_name = project_name + self._platform = platform + self._use_docker_build = use_docker_build + + def load_compose_json(self, deployment_dir: Path) -> dict: + env = os.environ.copy() + env["COMPOSE_PROFILES"] = "production" + p = subprocess.run( + ["docker", "compose", "-f", "docker-compose.yml", "config", "--format", "json"], + cwd=deployment_dir, + capture_output=True, + text=True, + env=env, + ) + if p.returncode != 0: + ui.error("docker compose config --format json failed (needs Docker Compose v2.10+)") + sys.exit(1) + return json.loads(p.stdout) + + def list_services(self, deployment_dir: Path) -> list[str]: + env = os.environ.copy() + env["COMPOSE_PROFILES"] = "production" + p = subprocess.run( + ["docker", "compose", "-f", "docker-compose.yml", "config", "--services"], + cwd=deployment_dir, + capture_output=True, + text=True, + env=env, + ) + if p.returncode != 0: + return [] + return [s.strip() for s in p.stdout.splitlines() if s.strip()] + + def collect_pushable(self, compose: dict, services: list[str]) -> list[PushableService]: + deployment_dir = self._paths.deployment_dir + project_root = self._paths.project_root + out: list[PushableService] = [] + svc_map = compose.get("services") or {} + for service in services: + spec = svc_map.get(service) + if not isinstance(spec, dict): + continue + build = spec.get("build") + if not isinstance(build, dict): + continue + image_tag = f"{self._project_name}-{service}:latest" + dockerfile_rel = (build.get("dockerfile") or "").strip() + context_rel = (build.get("context") or "").strip() + build_target = (build.get("target") or "").strip() + if not context_rel: + context_rel = ".." + if context_rel == "..": + build_context = project_root + elif context_rel.startswith("/"): + build_context = Path(context_rel) + else: + build_context = deployment_dir / context_rel + if dockerfile_rel: + if dockerfile_rel.startswith("/"): + dockerfile = Path(dockerfile_rel) + elif context_rel == ".." or build_context == project_root: + dockerfile = project_root / dockerfile_rel + else: + dockerfile = build_context / dockerfile_rel + else: + cand_a = deployment_dir / f"Dockerfile.{service}" + cand_b = deployment_dir / service / "Dockerfile" + if cand_a.is_file(): + dockerfile = cand_a + elif cand_b.is_file(): + dockerfile = cand_b + else: + ui.error(f"Could not determine Dockerfile for {service}") + sys.exit(1) + if not self._paths.input_hash_script.is_file(): + ui.error(f"Missing {self._paths.input_hash_script} (needed for dependency hashing)") + sys.exit(1) + h = compute_inputs_hash( + build_context, + dockerfile, + hash_script=self._paths.input_hash_script, + ) + if not h: + ui.error(f"Failed to compute input hash for {service}") + sys.exit(1) + out.append( + PushableService( + service=service, + image_tag=image_tag, + dockerfile=dockerfile, + build_context=build_context, + build_target=build_target, + input_hash=h, + ) + ) + return out + + def plan_builds(self, pushable: list[PushableService]) -> tuple[list[PushableService], list[str]]: + """Return (to_build, built_images_after) — built_images empty until build runs.""" + cache_root = self._paths.local_image_cache_dir + cache_root.mkdir(parents=True, exist_ok=True) + to_build: list[PushableService] = [] + for ps in pushable: + key = sanitize_ref(ps.image_tag) + cache_file = cache_root / key / "input.sha256" + prev = read_file_if_exists(cache_file).strip() + fp = local_image_layer_fp(ps.image_tag) + if prev and prev == ps.input_hash and fp: + continue + to_build.append(ps) + return to_build, [] + + def run_builds(self, to_build: list[PushableService]) -> list[str]: + if not to_build: + ui.success("Build skipped (no Docker inputs changed)") + return [] + ui.step(f"Building {len(to_build)} service(s)") + deployment_dir = self._paths.deployment_dir + env = os.environ.copy() + env["COMPOSE_PROJECT_NAME"] = self._project_name + env["COMPOSE_PROFILES"] = "production" + if self._use_docker_build: + cmd = [ + "docker", + "compose", + "-f", + "docker-compose.yml", + "--profile", + "production", + "build", + *[p.service for p in to_build], + ] + if subprocess.run(cmd, cwd=deployment_dir, env=env).returncode != 0: + ui.error("docker compose build failed") + sys.exit(1) + else: + for ps in to_build: + ui.substep(f"Building {ps.service} -> {ps.image_tag}...") + args = [ + "docker", + "buildx", + "build", + "--platform", + self._platform, + "--file", + str(ps.dockerfile), + "--tag", + ps.image_tag, + "--output=type=docker", + "--provenance=false", + "--sbom=false", + ] + if ps.build_target: + args.extend(["--target", ps.build_target]) + args.append(str(ps.build_context)) + if subprocess.run(args).returncode != 0: + ui.error(f"Build failed for {ps.service}") + sys.exit(1) + built: list[str] = [] + for ps in to_build: + key = sanitize_ref(ps.image_tag) + d = self._paths.local_image_cache_dir / key + d.mkdir(parents=True, exist_ok=True) + (d / "input.sha256").write_text(ps.input_hash, encoding="utf-8") + built.append(ps.image_tag) + ui.success(f"Build complete! {len(built)} image(s) built") + return built + + +def classify_push_and_external( + compose: dict, + project_name: str, + local_tags: set[str], + service_order: list[str], +) -> tuple[list[str], list[str]]: + services = compose.get("services") or {} + push_images: list[str] = [] + external: list[str] = [] + for name in service_order: + spec = services.get(name) + if not isinstance(spec, dict): + continue + image_from = (spec.get("image") or "").strip() + build = spec.get("build") + has_build = isinstance(build, dict) + if image_from: + if not has_build: + external.append(image_from) + else: + push_images.append(image_from) + else: + tag = f"{project_name}-{name}:latest" + if tag in local_tags: + push_images.append(tag) + return dedupe_preserve(push_images), dedupe_preserve(external) + + +def verify_built_subset_push(built: list[str], push_images: list[str], ui: object) -> None: + matching = sum(1 for bi in built if bi in push_images) + missing = [bi for bi in built if bi not in push_images] + not_built = [di for di in push_images if di not in built] + if len(built) != matching: + ui.error(f"Mismatch between built images ({len(built)}) and detected built images ({matching}).") + if missing: + print(f" Built but not detected: {' '.join(missing)}") + if not_built: + print(f" Detected but not built (external images): {' '.join(not_built)}") + print("Aborting to avoid pushing incorrect images.") + sys.exit(1) + + +def images_to_push_intersection(push_images: list[str], built: list[str]) -> list[str]: + out: list[str] = [] + for pi in push_images: + if pi in built: + out.append(pi) + return dedupe_preserve(out) diff --git a/scripts/deploy/config.py b/scripts/deploy/config.py new file mode 100644 index 0000000..68e406a --- /dev/null +++ b/scripts/deploy/config.py @@ -0,0 +1,70 @@ +"""Load deployment/.env and CLI into settings.""" + +from __future__ import annotations + +import os +import platform +import sys +from dataclasses import dataclass +from pathlib import Path + +from dotenv import load_dotenv + +from deploy.paths import ProjectPaths + + +@dataclass +class DeploySettings: + server: str + repo_name: str + deploy_path: str + platform: str + host_arch: str + platform_arch: str + use_docker_build: bool + paths: ProjectPaths + + +def _machine_arch() -> str: + m = platform.machine().lower() + if m in ("arm64", "aarch64"): + return "arm64" + if m in ("x86_64", "amd64", "i386", "i686"): + return "amd64" + return m + + +def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings: + if paths.env_file.is_file(): + load_dotenv(paths.env_file, override=False) + + server = (argv[1] if len(argv) > 1 else None) or os.environ.get("DEPLOYMENT_SERVER", "") + server = server.strip() + if not server: + sys.stderr.write( + "Server not specified. Usage: deploy.sh [user@host] [deployment_path] [platform]\n" + f" Or set DEPLOYMENT_SERVER in {paths.env_file} or as an environment variable\n\n" + "Example:\n" + " deploy.sh user@example.com /home/user/fromchat linux/arm64\n" + f" Or add to {paths.env_file}: DEPLOYMENT_SERVER=user@example.com\n" + ) + raise SystemExit(1) + + repo_name = "FromChat" + deploy_path = f"~/actions-runner/_work/{repo_name}/{repo_name}" + docker_platform = "linux/arm64" + + host_arch = _machine_arch() + platform_arch = docker_platform.split("/", 1)[-1] + use_docker_build = bool(host_arch and host_arch == platform_arch) + + return DeploySettings( + server=server, + repo_name=repo_name, + deploy_path=deploy_path, + platform=docker_platform, + host_arch=host_arch, + platform_arch=platform_arch, + use_docker_build=use_docker_build, + paths=paths, + ) diff --git a/scripts/deploy/docker_local.py b/scripts/deploy/docker_local.py new file mode 100644 index 0000000..657a6e1 --- /dev/null +++ b/scripts/deploy/docker_local.py @@ -0,0 +1,87 @@ +"""Local Docker daemon, Docker Desktop, and buildx setup.""" + +from __future__ import annotations + +import subprocess +import sys +import time + +import deploy.ui as ui + + +BUILDER_NAME = "fromchat-builder" + + +def ensure_daemon() -> None: + if _daemon_ok(): + return + ui.warning("Docker daemon is not running") + if not _start_desktop(): + ui.error("Failed to start Docker Desktop. Please start it manually and try again.") + sys.exit(1) + + +def _daemon_ok() -> bool: + return subprocess.run(["docker", "info"], capture_output=True).returncode == 0 + + +def _start_desktop() -> bool: + ui.substep("Starting Docker Desktop...") + if subprocess.run(["docker", "desktop", "start"], capture_output=True).returncode != 0: + return False + ui.substep("Waiting for Docker to start...", end="") + sys.stdout.flush() + max_wait = 60 + waited = 0 + while waited < max_wait: + if _daemon_ok(): + print() + return True + time.sleep(2) + waited += 2 + print(".", end="", flush=True) + print() + return False + + +def ensure_buildx(use_compose_build: bool) -> None: + if use_compose_build: + return + if subprocess.run(["docker", "buildx", "version"], capture_output=True).returncode != 0: + ui.error("Docker buildx not available. Install Docker Desktop.") + sys.exit(1) + _setup_builder() + + +def _setup_builder() -> None: + ui.step("Setting up buildx builder") + name = BUILDER_NAME + exists = subprocess.run(["docker", "buildx", "inspect", name], capture_output=True).returncode == 0 + if exists: + if subprocess.run(["docker", "buildx", "use", name], capture_output=True).returncode != 0: + ui.substep("Recreating builder...") + subprocess.run(["docker", "buildx", "rm", name], capture_output=True) + exists = False + elif subprocess.run(["docker", "buildx", "inspect", name], capture_output=True).returncode != 0: + ui.substep("Recreating builder (inspection failed)...") + subprocess.run(["docker", "buildx", "rm", name], capture_output=True) + exists = False + if not exists: + ui.substep("Creating builder with persistent cache...") + subprocess.run( + [ + "docker", + "buildx", + "create", + "--name", + name, + "--driver", + "docker-container", + "--driver-opt", + "image=moby/buildkit:latest", + "--use", + "--bootstrap", + ], + capture_output=True, + ) + subprocess.run(["docker", "buildx", "use", name], capture_output=True) diff --git a/scripts/deploy/main.py b/scripts/deploy/main.py new file mode 100644 index 0000000..883d13c --- /dev/null +++ b/scripts/deploy/main.py @@ -0,0 +1,90 @@ +"""CLI entry: build Docker images, pussh, rsync, restart remote systemd.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_SCRIPTS = Path(__file__).resolve().parent.parent +if str(_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_SCRIPTS)) + +from deploy.compose_build import ( # noqa: E402 + ComposeBuildPhase, + classify_push_and_external, + images_to_push_intersection, + remote_project_name, + verify_built_subset_push, +) +import deploy.ui as ui # noqa: E402 +from deploy.config import load_settings # noqa: E402 +import deploy.docker_local as docker_local # noqa: E402 +from deploy.paths import ProjectPaths # noqa: E402 +from deploy.ssh_auth import SshAuth # noqa: E402 +from deploy.transfer import DeployTransfer # noqa: E402 +from deploy.util import local_docker_image_tags # noqa: E402 + + +def main() -> None: + paths = ProjectPaths.from_deploy_package() + settings = load_settings(paths, sys.argv) + ui.banner() + creds = SshAuth(settings.server).authenticate() + + project_name = remote_project_name(settings.server, settings.deploy_path) + + ui.build_banner() + docker_local.ensure_daemon() + docker_local.ensure_buildx(settings.use_docker_build) + + ui.step("Detecting services") + build_phase = ComposeBuildPhase( + paths, + project_name=project_name, + platform=settings.platform, + use_docker_build=settings.use_docker_build, + ) + deployment_dir = paths.deployment_dir + services = build_phase.list_services(deployment_dir) + if not services: + ui.error("No services found in docker-compose.yml") + raise SystemExit(1) + + compose_json = build_phase.load_compose_json(deployment_dir) + pushable = build_phase.collect_pushable(compose_json, services) + to_build, _ = build_phase.plan_builds(pushable) + built_images = build_phase.run_builds(to_build) + + ui.deploy_banner(settings.server) + + transfer = DeployTransfer(paths) + transfer.ensure_pussh() + + push_images, external_images = classify_push_and_external( + compose_json, + project_name, + local_docker_image_tags(), + services, + ) + + verify_built_subset_push(built_images, push_images, ui) + + if not push_images and not external_images: + ui.error(f"No images found in docker-compose.yml or built locally for project {project_name}") + raise SystemExit(1) + + to_push = images_to_push_intersection(push_images, built_images) + transfer.pussh_images(creds, to_push) + transfer.pull_external_on_server(creds, external_images) + + transfer.rsync_deployment(creds, settings.deploy_path) + transfer.copy_env_prod(creds, settings.deploy_path) + deploy_resolved = transfer.sync_firebase_cert(creds, settings.deploy_path) + transfer.run_remote_systemd(creds, deploy_resolved) + + print() + ui.success("Deployment complete!") + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy/paths.py b/scripts/deploy/paths.py new file mode 100644 index 0000000..86abf22 --- /dev/null +++ b/scripts/deploy/paths.py @@ -0,0 +1,35 @@ +"""Resolved filesystem paths for the Web repo.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class ProjectPaths: + """Root and well-known directories (Web repo root = project root).""" + + project_root: Path + scripts_dir: Path + deployment_dir: Path + env_file: Path + local_cache_root: Path + local_image_cache_dir: Path + input_hash_script: Path + + @classmethod + def from_deploy_package(cls) -> ProjectPaths: + deploy_dir = Path(__file__).resolve().parent + scripts_dir = deploy_dir.parent + project_root = scripts_dir.parent + deployment_dir = project_root / "deployment" + return cls( + project_root=project_root, + scripts_dir=scripts_dir, + deployment_dir=deployment_dir, + env_file=deployment_dir / ".env", + local_cache_root=project_root / ".deploy-cache", + local_image_cache_dir=project_root / ".deploy-cache" / "images", + input_hash_script=scripts_dir / "docker_inputs_hash.py", + ) diff --git a/scripts/deploy/ssh_auth.py b/scripts/deploy/ssh_auth.py new file mode 100644 index 0000000..6c190c3 --- /dev/null +++ b/scripts/deploy/ssh_auth.py @@ -0,0 +1,109 @@ +"""SSH key agent and optional sudo password for remote.""" + +from __future__ import annotations + +import getpass +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import deploy.ui as ui + + +@dataclass +class SshCredentials: + server: str + sudo_password: str + + +class SshAuth: + def __init__(self, server: str) -> None: + self._server = server + + def authenticate(self) -> SshCredentials: + ui.step("Authentication") + self._ensure_agent() + key_file = Path.home() / ".ssh" / "id_rsa" + self._ensure_key_file(key_file) + self._ensure_key_in_agent(key_file) + self._verify_key_auth(key_file) + sudo_password = self._prompt_sudo() + return SshCredentials(server=self._server, sudo_password=sudo_password) + + def _ensure_agent(self) -> None: + if os.environ.get("SSH_AUTH_SOCK"): + return + subprocess.run(["ssh-agent", "-s"], capture_output=True, check=False) + + def _ensure_key_file(self, key_file: Path) -> None: + if not key_file.is_file(): + ui.error(f"SSH key not found at {key_file}") + sys.stderr.write( + " Please generate an SSH key pair first:\n" + " ssh-keygen -t rsa -b 4096 -C 'your_email@example.com'\n" + ) + raise SystemExit(1) + + def _ensure_key_in_agent(self, key_file: Path) -> None: + loaded = False + r = subprocess.run(["ssh-add", "-l"], capture_output=True, text=True) + if r.returncode == 0: + fp_r = subprocess.run( + ["ssh-keygen", "-lf", str(key_file)], + capture_output=True, + text=True, + ) + if fp_r.returncode == 0: + parts = fp_r.stdout.strip().split() + fingerprint = parts[1] if len(parts) > 1 else "" + if fingerprint and fingerprint in r.stdout: + loaded = True + if not loaded: + ui.substep("Adding SSH key to agent...") + if subprocess.run(["ssh-add", str(key_file)], capture_output=True).returncode != 0: + ui.error("Failed to add SSH key to agent. Check your key passphrase.") + raise SystemExit(1) + + def _verify_key_auth(self, key_file: Path) -> None: + pub = key_file.with_suffix(key_file.suffix + ".pub") + ok = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + "-o", + "StrictHostKeyChecking=no", + self._server, + "echo 'SSH key works'", + ], + capture_output=True, + ).returncode + if ok == 0: + return + ui.error(f"SSH key authentication failed for {self._server}") + sys.stderr.write( + f' Copy your public key to the server, then re-run deploy:\n ssh-copy-id -i "{pub}" "{self._server}"\n\n' + " Or manually append this key to ~/.ssh/authorized_keys on the server:\n" + ) + if pub.is_file(): + sys.stderr.write(f" {pub.read_text(encoding='utf-8', errors='replace').strip()}\n") + raise SystemExit(1) + + def _prompt_sudo(self) -> str: + while True: + pw = getpass.getpass(" • Sudo password: ") + if not pw: + ui.warning("No password provided - assuming passwordless sudo") + return "" + chk = subprocess.run( + ["ssh", self._server, "sudo", "-S", "-v"], + input=(pw + "\n").encode(), + capture_output=True, + ) + if chk.returncode == 0: + return pw + ui.error("Invalid password, please try again") diff --git a/scripts/deploy/transfer.py b/scripts/deploy/transfer.py new file mode 100644 index 0000000..e8206ab --- /dev/null +++ b/scripts/deploy/transfer.py @@ -0,0 +1,288 @@ +"""Image pussh, rsync deployment, Firebase cert, remote systemd.""" + +from __future__ import annotations + +import shlex +import subprocess +import sys +import tempfile +from pathlib import Path + +from deploy.paths import ProjectPaths +from deploy.ssh_auth import SshCredentials +import deploy.ui as ui + +UNREGISTRY_IMAGE = "ghcr.io/psviderski/unregistry" + +REMOTE_SYSTEMD_SCRIPT = r"""set -e + +REMOTE_SUDO_PASS="${SUDO_PASSWORD:-}" +REMOTE_DEPLOY_PATH="${DEPLOY_PATH:-}" +export SUDO_PROMPT="" + +sudo_cmd() { + if [ -n "$REMOTE_SUDO_PASS" ]; then + echo "$REMOTE_SUDO_PASS" | sudo -S -p '' "$@" 2>/dev/null + else + sudo "$@" 2>/dev/null + fi +} + +if [ -z "$REMOTE_DEPLOY_PATH" ]; then + echo "❌ DEPLOY_PATH is not set" + exit 1 +fi + +mkdir -p "$REMOTE_DEPLOY_PATH/deployment" "$REMOTE_DEPLOY_PATH/backend" +cd "$REMOTE_DEPLOY_PATH/deployment" + +if [ ! -f "$REMOTE_DEPLOY_PATH/deployment/.env" ]; then + echo "⚠️ Warning: .env file not found" +fi + +if systemctl is-active --quiet fromchat; then + sudo_cmd systemctl stop fromchat +fi + +COMPOSE_PROFILES=production docker compose down --remove-orphans > /dev/null 2>&1 || true + +sudo_cmd cp -f "$REMOTE_DEPLOY_PATH/deployment/fromchat.service" /etc/systemd/system/fromchat.service +sudo_cmd systemctl daemon-reload +sudo_cmd systemctl restart fromchat + +sleep 3 +if ! systemctl is-active --quiet fromchat; then + echo "❌ Service failed to start" + sudo_cmd journalctl --no-pager -xeu fromchat -n 30 + exit 1 +fi +""" + + +class DeployTransfer: + def __init__(self, paths: ProjectPaths) -> None: + self._paths = paths + + def ensure_pussh(self) -> None: + if subprocess.run(["docker", "pussh", "--help"], capture_output=True).returncode != 0: + ui.error("docker pussh plugin not installed") + print(" Install: npm run install:pussh") + + def ensure_unregistry(self, creds: SshCredentials) -> None: + check = ( + "sudo docker images --format '{{.Repository}}:{{.Tag}}' | " + f"grep -q '^{UNREGISTRY_IMAGE}$'" + ) + if subprocess.run(["ssh", creds.server, check], capture_output=True).returncode == 0: + return + ui.substep("Pulling unregistry image (one-time setup)...") + if creds.sudo_password: + inner = f"echo {shlex.quote(creds.sudo_password)} | sudo -S -p '' docker pull {UNREGISTRY_IMAGE}" + else: + inner = f"sudo docker pull {UNREGISTRY_IMAGE}" + subprocess.run(["ssh", creds.server, inner]) + + def pussh_images(self, creds: SshCredentials, images: list[str]) -> None: + ui.step("Transferring images") + if not images: + ui.success("Skipping image push (nothing was rebuilt this run)") + return + self.ensure_unregistry(creds) + for image in images: + ui.substep(f"Pushing {image}...") + if subprocess.run(["docker", "pussh", image, creds.server]).returncode != 0: + ui.error(f"Failed to push {image}") + raise SystemExit(1) + print() + + def pull_external_on_server(self, creds: SshCredentials, images: list[str]) -> None: + if not images: + return + ui.step("Pulling external images on server") + for image in images: + ui.substep(f"Pulling {image}...") + if creds.sudo_password: + inner = f"echo {shlex.quote(creds.sudo_password)} | sudo -S -p '' docker pull {shlex.quote(image)}" + else: + inner = f"sudo docker pull {shlex.quote(image)}" + if subprocess.run(["ssh", creds.server, inner]).returncode != 0: + ui.error(f"Failed to pull {image} on server") + raise SystemExit(1) + print() + + def prepare_remote_dirs(self, creds: SshCredentials, deploy_path: str) -> None: + dp = deploy_path + d_dep = shlex.quote(f"{dp}/deployment") + d_back = shlex.quote(f"{dp}/backend") + if creds.sudo_password: + pw = shlex.quote(creds.sudo_password) + script = f"""set -e +echo {pw} | sudo -S -p '' mkdir -p {d_dep} {d_back} 2>/dev/null || true +echo {pw} | sudo -S -p '' chown -R $(whoami):$(whoami) {d_dep} {d_back} 2>/dev/null || true +""" + subprocess.run(["ssh", creds.server, "bash"], input=script.encode(), capture_output=True) + else: + subprocess.run( + [ + "ssh", + creds.server, + f"sudo mkdir -p {d_dep} {d_back} && sudo chown -R $(whoami):$(whoami) {d_dep} {d_back}", + ], + capture_output=True, + ) + + def rsync_deployment(self, creds: SshCredentials, deploy_path: str) -> None: + ui.step("Transferring deployment files") + self.prepare_remote_dirs(creds, deploy_path) + project_root = self._paths.project_root + deployment_dir = self._paths.deployment_dir + ui.substep("Copying deployment directory...") + gl = subprocess.run( + ["git", "ls-files", "--others", "--ignored", "--exclude-standard", "deployment/"], + cwd=project_root, + capture_output=True, + text=True, + ) + lines = [ln.replace("deployment/", "", 1) for ln in gl.stdout.splitlines() if ln.strip()] + with tempfile.NamedTemporaryFile("w", suffix="-rsync-exclude", delete=False, encoding="utf-8") as tf: + exclude_path = Path(tf.name) + tf.write("\n".join(lines)) + try: + rsync = subprocess.run( + [ + "rsync", + "-avz", + "--delete", + f"--exclude-from={exclude_path}", + f"{deployment_dir}/", + f"{creds.server}:{deploy_path}/deployment/", + ], + cwd=project_root, + capture_output=True, + text=True, + ) + if rsync.returncode != 0: + ui.error("Rsync failed. Error output:") + for line in (rsync.stderr or rsync.stdout or "").splitlines(): + print(f" {line}") + ui.error("Failed to copy deployment directory") + raise SystemExit(1) + finally: + exclude_path.unlink(missing_ok=True) + + def copy_env_prod(self, creds: SshCredentials, deploy_path: str) -> None: + prod = self._paths.deployment_dir / ".env.prod" + if prod.is_file(): + ui.substep("Copying .env.prod to .env...") + if subprocess.run(["scp", str(prod), f"{creds.server}:{deploy_path}/deployment/.env"], capture_output=True).returncode != 0: + ui.warning("Failed to copy .env.prod to .env") + else: + ui.warning(".env.prod not found in deployment directory") + + def resolve_deploy_path_on_server(self, server: str, deploy_path: str) -> str: + r = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", server, f"eval echo {deploy_path}"], + capture_output=True, + text=True, + ) + out = r.stdout.strip() + return out if out else deploy_path + + def firebase_cert_path(self) -> Path: + return self._paths.project_root / "backend" / "firebase-cert.json" + + def cleanup_remote_firebase_dir(self, creds: SshCredentials, deploy_path_resolved: str) -> None: + d = deploy_path_resolved + if creds.sudo_password: + pw = shlex.quote(creds.sudo_password) + script = f"""set -e +D={shlex.quote(d)} +C="$D/backend/firebase-cert.json" +mkdir -p "$D/backend" 2>/dev/null || true +if [ -d "$C" ]; then + echo {pw} | sudo -S -p '' rm -rf "$C" +fi +echo {pw} | sudo -S -p '' chown -R "$(whoami):$(whoami)" "$D/backend" 2>/dev/null || true +""" + subprocess.run(["ssh", creds.server, "bash"], input=script.encode(), capture_output=True) + else: + q = shlex.quote(d) + subprocess.run( + [ + "ssh", + creds.server, + f"D={q}; C=\"$D/backend/firebase-cert.json\"; mkdir -p \"$D/backend\"; " + f'if [ -d "$C" ]; then sudo rm -rf "$C" 2>/dev/null || rm -rf "$C"; fi; ' + f'sudo chown -R $(whoami):$(whoami) "$D/backend" 2>/dev/null || true', + ], + capture_output=True, + ) + + def _wait_firebase_loop(self, cert: Path) -> None: + while True: + if cert.is_file(): + return + if cert.is_dir(): + print( + f" ⚠ {cert} is a directory. Delete it and save the Firebase service account JSON as a file at that exact path." + ) + elif cert.exists(): + print(f" ⚠ {cert} exists but is not a regular file.") + else: + print(f" ⚠ Missing {cert} (Firebase service account JSON for FCM).") + print(" Fix this, then press Enter to check again (Ctrl+C to abort deploy).") + input() + + def sync_firebase_cert(self, creds: SshCredentials, deploy_path: str) -> str: + ui.substep( + "Firebase service account (runtime bind-mount: backend/firebase-cert.json)..." + ) + resolved = self.resolve_deploy_path_on_server(creds.server, deploy_path) + self.cleanup_remote_firebase_dir(creds, resolved) + cert = self.firebase_cert_path() + self._wait_firebase_loop(cert) + remote = f"{resolved}/backend/firebase-cert.json" + self.scp_firebase(creds, cert, remote) + return resolved + + def scp_firebase(self, creds: SshCredentials, cert: Path, remote_path: str) -> None: + ui.substep("Copying backend/firebase-cert.json...") + r = subprocess.run( + ["scp", str(cert), f"{creds.server}:{remote_path}"], + capture_output=True, + text=True, + ) + if r.returncode != 0: + ui.error("Failed to copy firebase-cert.json to server") + print(f" Target: {creds.server}:{remote_path}", file=sys.stderr) + err = (r.stderr or r.stdout or "").strip() + if err: + for line in err.splitlines(): + print(f" {line}", file=sys.stderr) + else: + print(" (scp produced no output.)", file=sys.stderr) + raise SystemExit(1) + subprocess.run( + ["ssh", creds.server, f"chmod 600 {shlex.quote(remote_path)}"], + capture_output=True, + ) + t = subprocess.run( + ["ssh", creds.server, f"test -f {shlex.quote(remote_path)}"], + capture_output=True, + ) + if t.returncode != 0: + ui.error(f"Server path is not a regular file after copy: {remote_path}") + raise SystemExit(1) + + def run_remote_systemd(self, creds: SshCredentials, deploy_path_resolved: str) -> None: + ui.step("Deploying on server") + pw = creds.sudo_password + dp = deploy_path_resolved + remote_cmd = f"SUDO_PASSWORD={shlex.quote(pw)} DEPLOY_PATH={shlex.quote(dp)} bash -s" + r = subprocess.run( + ["ssh", creds.server, remote_cmd], + input=REMOTE_SYSTEMD_SCRIPT.encode(), + text=False, + ) + if r.returncode != 0: + raise SystemExit(r.returncode) diff --git a/scripts/deploy/ui.py b/scripts/deploy/ui.py new file mode 100644 index 0000000..15fdc2f --- /dev/null +++ b/scripts/deploy/ui.py @@ -0,0 +1,49 @@ +"""Terminal output (Rich), matching previous deploy.sh style.""" + +from __future__ import annotations + +from rich.console import Console +from rich.text import Text + +_console = Console(highlight=False) + +def banner() -> None: + _console.print() + _console.print(Text("🚀 Deployment", style="bold magenta")) + _console.print() + + +def build_banner() -> None: + _console.print() + _console.print(Text("🔨 Building Docker images", style="bold magenta")) + _console.print() + + +def deploy_banner(server: str) -> None: + _console.print() + _console.print(Text(f"🚀 Deploying to {server}", style="bold magenta")) + _console.print() + + +def info(msg: str) -> None: + _console.print(Text("ℹ ", style="blue"), msg, sep="") + + +def success(msg: str) -> None: + _console.print(Text("✓ ", style="green"), msg, sep="") + + +def warning(msg: str) -> None: + _console.print(Text("⚠ ", style="yellow"), msg, sep="") + + +def error(msg: str) -> None: + _console.print(Text("✗ ", style="red"), msg, sep="") + + +def step(msg: str) -> None: + _console.print(Text("→ ", style="bold cyan"), Text(msg, style="bold"), sep="") + + +def substep(msg: str, *, end: str = "\n") -> None: + _console.print(Text(" • ", style="green"), msg, sep="", end=end) diff --git a/scripts/deploy/util.py b/scripts/deploy/util.py new file mode 100644 index 0000000..e46d86d --- /dev/null +++ b/scripts/deploy/util.py @@ -0,0 +1,85 @@ +"""Small helpers: hashing, dedupe, cache keys.""" + +from __future__ import annotations + +import hashlib +import subprocess +import sys +from pathlib import Path + + +def sanitize_ref(ref: str) -> str: + s = ref.replace("/", "_").replace(":", "__").replace("@", "__at__") + return s + + +def dedupe_preserve(items: list[str]) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for x in items: + if x not in seen: + seen.add(x) + out.append(x) + return out + + +def read_file_if_exists(path: Path) -> str: + if path.is_file(): + return path.read_text(encoding="utf-8", errors="replace") + return "" + + +def local_image_layer_fp(image: str) -> str: + def inspect_layers(ref: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", "image", "inspect", "-f", "{{json .RootFS.Layers}}", ref], + capture_output=True, + text=True, + ) + + p = inspect_layers(image) + if p.returncode != 0: + # Docker Desktop occasionally ends up in a state where repo:tag exists in `docker images` + # but `docker image inspect repo:tag` fails. Inspecting by content-addressed ID works. + id_p = subprocess.run( + ["docker", "images", "--no-trunc", "--format", "{{.ID}}", image], + capture_output=True, + text=True, + ) + image_id = (id_p.stdout or "").strip() + if not image_id: + return "" + p = inspect_layers(image_id) + if p.returncode != 0: + return "" + + return hashlib.sha256(p.stdout.encode()).hexdigest() + + +def compute_inputs_hash( + context: Path, + dockerfile: Path, + *, + hash_script: Path, + python_exe: str | None = None, +) -> str: + exe = python_exe or sys.executable + p = subprocess.run( + [exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)], + capture_output=True, + text=True, + ) + if p.returncode != 0: + return "" + return p.stdout.strip() + + +def local_docker_image_tags() -> set[str]: + p = subprocess.run( + ["docker", "images", "--format", "{{.Repository}}:{{.Tag}}"], + capture_output=True, + text=True, + ) + if p.returncode != 0: + return set() + return {line.strip() for line in p.stdout.splitlines() if line.strip()}