diff --git a/README.md b/README.md index 30d74d2..6c2ecf1 100644 --- a/README.md +++ b/README.md @@ -1 +1,6 @@ -test \ No newline at end of file +X-Real-IP: "$remote_addr" +X-Forwarded-For: "$proxy_add_x_forwarded_for" +X-Forwarded-Proto: "$scheme" +X-Forwarded-Host: "$host" + + diff --git a/wallarm-ct-deploy.sh b/wallarm-ct-deploy.sh index 637edba..176b99a 100755 --- a/wallarm-ct-deploy.sh +++ b/wallarm-ct-deploy.sh @@ -57,7 +57,20 @@ else CURL_INSECURE_FLAG="" fi -# Internal registry endpoints (from stealth deployment) +# GitLab artifact URLs (primary source) +GITLAB_BASE_URL="https://git.sechpoint.app/customer-engineering/wallarm" +GITLAB_RAW_URL="https://git.sechpoint.app/customer-engineering/wallarm/-/raw/main" +GITLAB_DOCKER_BINARY_URL="${GITLAB_RAW_URL}/binaries/docker-29.2.1.tgz" +GITLAB_DOCKER_CHECKSUM_URL="${GITLAB_RAW_URL}/binaries/docker-29.2.1.tgz.sha256" +GITLAB_WALLARM_IMAGE_URL="${GITLAB_RAW_URL}/images/wallarm-node-6.11.0-rc1.tar.gz" +GITLAB_WALLARM_CHECKSUM_URL="${GITLAB_RAW_URL}/images/wallarm-node-6.11.0-rc1.tar.gz.sha256" + +# Local artifact directories (relative to script location) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOCAL_BINARY_DIR="${SCRIPT_DIR}/binaries" +LOCAL_IMAGE_DIR="${SCRIPT_DIR}/images" + +# Internal registry endpoints (from stealth deployment) - fallback source INTERNAL_DOCKER_REGISTRY="https://deployment:elqXBsyT4BGXPYPeD07or8hT0Lb9Lpf@hub.ct.sechpoint.app" INTERNAL_DOCKER_DOWNLOAD="https://deployment:elqXBsyT4BGXPYPeD07or8hT0Lb9Lpf@ct.sechpoint.app" # Extracted hostnames (without credentials) for Docker operations @@ -126,6 +139,96 @@ fail_with_remediation() { exit 1 } +# ============================================================================== +# GITLAB ARTIFACT FUNCTIONS +# ============================================================================== + +download_from_gitlab() { + local url="$1" + local output_path="$2" + local description="$3" + + log_message "INFO" "Attempting to download $description from GitLab..." + log_message "DEBUG" "URL: $url" + log_message "DEBUG" "Output path: $output_path" + + # Use curl with follow redirects, fail on HTTP error, timeout settings + if curl -fL "$CURL_INSECURE_FLAG" --connect-timeout 30 --max-time 300 --progress-bar "$url" -o "$output_path"; then + log_message "SUCCESS" "Downloaded $description to $output_path" + return 0 + else + local curl_exit=$? + log_message "ERROR" "Failed to download $description from GitLab (curl exit: $curl_exit)" + # Clean up partial download if it exists + if [ -f "$output_path" ]; then + rm -f "$output_path" + log_message "DEBUG" "Removed partial download: $output_path" + fi + return 1 + fi +} + +verify_checksum() { + local file_path="$1" + local checksum_file_or_url="$2" + local description="$3" + + log_message "INFO" "Verifying $description checksum..." + + local checksum_file="" + # If checksum is a URL, download it first + if [[ "$checksum_file_or_url" =~ ^https?:// ]]; then + checksum_file="/tmp/$(basename "$checksum_file_or_url")" + log_message "DEBUG" "Downloading checksum from URL: $checksum_file_or_url" + if ! curl -fL "$CURL_INSECURE_FLAG" --connect-timeout 10 --max-time 30 -s "$checksum_file_or_url" -o "$checksum_file"; then + log_message "WARNING" "Could not download checksum file, skipping verification" + return 0 # Skip verification if checksum can't be downloaded + fi + else + checksum_file="$checksum_file_or_url" + fi + + # Verify checksum file exists + if [ ! -f "$checksum_file" ]; then + log_message "WARNING" "Checksum file not found: $checksum_file, skipping verification" + return 0 + fi + + # Get expected checksum (first field from checksum file) + local expected_checksum + expected_checksum=$(awk '{print $1}' "$checksum_file" 2>/dev/null) + if [ -z "$expected_checksum" ]; then + log_message "WARNING" "Could not read checksum from $checksum_file, skipping verification" + return 0 + fi + + # Compute actual checksum + log_message "DEBUG" "Computing SHA256 checksum of $file_path..." + local actual_checksum + if command -v sha256sum >/dev/null 2>&1; then + actual_checksum=$(sha256sum "$file_path" | awk '{print $1}') + elif command -v shasum >/dev/null 2>&1; then + actual_checksum=$(shasum -a 256 "$file_path" | awk '{print $1}') + else + log_message "WARNING" "sha256sum or shasum not available, skipping checksum verification" + return 0 + fi + + # Compare checksums + if [ "$expected_checksum" = "$actual_checksum" ]; then + log_message "SUCCESS" "$description checksum verified successfully" + return 0 + else + log_message "ERROR" "$description checksum verification FAILED" + log_message "DEBUG" "Expected: $expected_checksum" + log_message "DEBUG" "Actual: $actual_checksum" + # Clean up corrupted file + rm -f "$file_path" + log_message "INFO" "Removed corrupted file: $file_path" + return 1 + fi +} + # ============================================================================== # PREFLIGHT CHECK VERIFICATION # ============================================================================== @@ -200,6 +303,27 @@ verify_preflight_check() { log_message "SUCCESS" " Registry Reachable: $REGISTRY_REACHABLE" log_message "SUCCESS" " Download Reachable: $DOWNLOAD_REACHABLE" + # Check for local artifact directories + if [ -d "$LOCAL_BINARY_DIR" ]; then + log_message "INFO" " Local binaries directory: $LOCAL_BINARY_DIR (exists)" + local binary_count=$(ls "$LOCAL_BINARY_DIR"/*.tgz 2>/dev/null | wc -l) + if [ "$binary_count" -gt 0 ]; then + log_message "INFO" " Found $binary_count Docker binary file(s)" + fi + else + log_message "INFO" " Local binaries directory: $LOCAL_BINARY_DIR (not found)" + fi + + if [ -d "$LOCAL_IMAGE_DIR" ]; then + log_message "INFO" " Local images directory: $LOCAL_IMAGE_DIR (exists)" + local image_count=$(ls "$LOCAL_IMAGE_DIR"/*.tar.gz 2>/dev/null | wc -l) + if [ "$image_count" -gt 0 ]; then + log_message "INFO" " Found $image_count Wallarm image file(s)" + fi + else + log_message "INFO" " Local images directory: $LOCAL_IMAGE_DIR (not found)" + fi + # Validate we have at least one cloud region reachable if [ "$US_CLOUD_REACHABLE" = "false" ] && [ "$EU_CLOUD_REACHABLE" = "false" ]; then fail_with_remediation "No Wallarm cloud region reachable" \ @@ -519,17 +643,71 @@ setup_docker_engine() { log_message "INFO" "Docker not found or not running. Proceeding with installation..." - # Determine binary source + # Determine binary source (priority: GitLab -> local dir -> current dir -> internal proxy) local binary_file="docker-$DOCKER_VERSION.tgz" local binary_path="" - if [ "$DOWNLOAD_REACHABLE" = "true" ]; then + # 1. Try GitLab download (primary source) + log_message "INFO" "Attempting to download Docker binary from GitLab..." + if download_from_gitlab "$GITLAB_DOCKER_BINARY_URL" "$binary_file" "Docker binary"; then + if verify_checksum "$binary_file" "$GITLAB_DOCKER_CHECKSUM_URL" "Docker binary"; then + binary_path="$binary_file" + log_message "SUCCESS" "Docker binary downloaded from GitLab and checksum verified" + else + log_message "WARNING" "GitLab Docker binary checksum verification failed, trying other sources" + # Remove corrupted download + rm -f "$binary_file" + fi + fi + + # 2. Check local binaries directory + if [ -z "$binary_path" ] && [ -d "$LOCAL_BINARY_DIR" ]; then + log_message "INFO" "Checking local binaries directory: $LOCAL_BINARY_DIR" + local local_binary="$LOCAL_BINARY_DIR/docker-29.2.1.tgz" + local local_checksum="$LOCAL_BINARY_DIR/docker-29.2.1.tgz.sha256" + + if [ -f "$local_binary" ]; then + log_message "INFO" "Found local Docker binary: $local_binary" + # Copy to current directory for consistency with extraction logic + cp "$local_binary" "$binary_file" + if verify_checksum "$binary_file" "$local_checksum" "local Docker binary"; then + binary_path="$binary_file" + log_message "SUCCESS" "Using local Docker binary from binaries directory" + else + log_message "WARNING" "Local Docker binary checksum verification failed" + rm -f "$binary_file" + fi + fi + fi + + # 3. Check current directory for any docker-*.tgz (existing fallback) + if [ -z "$binary_path" ]; then + log_message "INFO" "Checking current directory for Docker binaries..." + local local_files + local_files=$(ls docker-*.tgz 2>/dev/null | head -1) + if [ -n "$local_files" ]; then + binary_path="$local_files" + log_message "SUCCESS" "Using local Docker binary: $binary_path" + # Optional: Try to verify checksum if .sha256 file exists + local checksum_file="${local_files}.sha256" + if [ -f "$checksum_file" ]; then + if verify_checksum "$binary_path" "$checksum_file" "Docker binary"; then + log_message "SUCCESS" "Local Docker binary checksum verified" + else + log_message "WARNING" "Local Docker binary checksum verification failed, but continuing" + fi + fi + fi + fi + + # 4. Try internal proxy (if reachable per preflight check) + if [ -z "$binary_path" ] && [ "$DOWNLOAD_REACHABLE" = "true" ]; then # Download Docker static binary from internal server - log_message "INFO" "Downloading Docker static binary for $ARCHITECTURE..." + log_message "INFO" "Downloading Docker static binary for $ARCHITECTURE from internal proxy..." local download_url="$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz" if curl -fL $CURL_INSECURE_FLAG --connect-timeout 30 "$download_url" -o "$binary_file"; then - log_message "SUCCESS" "Downloaded Docker binary: $binary_file" + log_message "SUCCESS" "Downloaded Docker binary from internal proxy: $binary_file" binary_path="$binary_file" else log_message "ERROR" "Failed to download Docker binary from $download_url" @@ -537,22 +715,17 @@ setup_docker_engine() { fi fi - # Fallback: Check for local Docker binary + # 5. Final fallback: no binary available if [ -z "$binary_path" ]; then - log_message "INFO" "Checking for local Docker binary..." - local local_files - local_files=$(ls docker-*.tgz 2>/dev/null | head -1) - if [ -n "$local_files" ]; then - binary_path="$local_files" - log_message "SUCCESS" "Using local Docker binary: $binary_path" - else - fail_with_remediation "No Docker binary available" \ - "Please provide a Docker static binary: -1. Download manually: - curl -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o docker.tgz -2. Or place an existing docker-*.tgz file in current directory -3. Re-run the script after downloading" - fi + fail_with_remediation "No Docker binary available" \ + "Please provide a Docker static binary using one of these methods: +1. GitLab (primary): Ensure network access to $GITLAB_BASE_URL +2. Local binaries directory: Place docker-29.2.1.tgz and .sha256 in $LOCAL_BINARY_DIR/ +3. Current directory: Place any docker-*.tgz file in current directory +4. Internal proxy: Ensure network access to $DOCKER_DOWNLOAD_HOST + Download manually: curl -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o docker.tgz + +Re-run the script after providing the binary." fi # Extract and install @@ -640,11 +813,12 @@ Possible solutions: Steps to fix: 1. Delete corrupted file: rm -f docker-*.tgz 2. Check disk space: df -h . -3. Download manually and verify: - curl -v -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o test.tgz - file test.tgz - tar -tzf test.tgz -4. Check if tar command works: tar --version" +3. Try alternative sources: + a) GitLab: curl -L '$GITLAB_DOCKER_BINARY_URL' -o docker.tgz + b) Local directory: Check $LOCAL_BINARY_DIR/docker-29.2.1.tgz + c) Internal proxy: curl -v -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o test.tgz +4. Verify downloaded file: file test.tgz && tar -tzf test.tgz +5. Check if tar command works: tar --version" fi log_message "SUCCESS" "Tar archive validation passed" @@ -707,8 +881,12 @@ Check the binary: sudo /usr/bin/dockerd --version The Docker static binary might be for wrong architecture or corrupted. -Try downloading manually: - curl -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o docker.tgz +Try downloading manually from one of these sources: +1. GitLab: curl -L '$GITLAB_DOCKER_BINARY_URL' -o docker.tgz +2. Local directory: Check $LOCAL_BINARY_DIR/docker-29.2.1.tgz +3. Internal proxy: curl -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o docker.tgz + +Then extract and install: tar xzvf docker.tgz sudo cp docker/* /usr/bin/" else @@ -1170,48 +1348,129 @@ Check for Docker logs: deploy_wallarm_node() { log_message "INFO" "Deploying Wallarm filtering node..." - # Pull Wallarm Docker image - log_message "INFO" "Pulling Wallarm Docker image from internal registry: $WALLARM_IMAGE_SOURCE" + # Load Wallarm Docker image (priority: GitLab -> local dir -> current dir -> internal registry) + log_message "INFO" "Loading Wallarm Docker image..." + local image_loaded=false - if [ "$REGISTRY_REACHABLE" = "true" ]; then - if ! sudo docker pull "$WALLARM_IMAGE_SOURCE"; then - fail_with_remediation "Failed to pull Wallarm image from internal registry" \ - "Docker pull from internal registry failed. Possible reasons: -1. Network connectivity to $DOCKER_REGISTRY_HOST -2. Authentication required for internal registry -3. Insufficient disk space - -Solutions: -1. Check network: curl -I $INTERNAL_DOCKER_REGISTRY -2. Login to internal registry if required -3. Use local image fallback: docker save/load -4. Check disk: df -h /var/lib/docker" - fi - - # Re-tag to standard name - sudo docker tag "$WALLARM_IMAGE_SOURCE" "$WALLARM_IMAGE_TARGET" - log_message "SUCCESS" "Wallarm image pulled and tagged successfully" - else - # Use local image - log_message "INFO" "Using local Wallarm image (registry not reachable)" - local local_image - local_image=$(ls wallarm-node-*.tar 2>/dev/null | head -1) - if [ -n "$local_image" ]; then - if ! sudo docker load -i "$local_image"; then - fail_with_remediation "Failed to load local Wallarm image" \ - "Local Wallarm image file may be corrupted: -1. Verify file integrity: tar -tzf wallarm-node-*.tar -2. Download a fresh image on another machine: - docker pull $WALLARM_IMAGE_SOURCE - docker save $WALLARM_IMAGE_TARGET -o wallarm-node-latest.tar -3. Copy the file to this machine and re-run" + # 1. Try GitLab download (primary source) + local gitlab_image_file="wallarm-node-6.11.0-rc1.tar.gz" + if [ "$image_loaded" = "false" ]; then + log_message "INFO" "Attempting to download Wallarm image from GitLab..." + if download_from_gitlab "$GITLAB_WALLARM_IMAGE_URL" "$gitlab_image_file" "Wallarm Docker image"; then + if verify_checksum "$gitlab_image_file" "$GITLAB_WALLARM_CHECKSUM_URL" "Wallarm Docker image"; then + log_message "INFO" "Loading Wallarm image from GitLab download..." + if gunzip -c "$gitlab_image_file" | sudo docker load; then + log_message "SUCCESS" "Wallarm image loaded from GitLab download" + image_loaded=true + else + log_message "ERROR" "Failed to load Wallarm image from GitLab download" + fi + # Cleanup downloaded file + rm -f "$gitlab_image_file" + else + log_message "WARNING" "GitLab Wallarm image checksum verification failed" + rm -f "$gitlab_image_file" fi - log_message "SUCCESS" "Local Wallarm image loaded successfully" + fi + fi + + # 2. Check local images directory + if [ "$image_loaded" = "false" ] && [ -d "$LOCAL_IMAGE_DIR" ]; then + log_message "INFO" "Checking local images directory: $LOCAL_IMAGE_DIR" + local local_image="$LOCAL_IMAGE_DIR/wallarm-node-6.11.0-rc1.tar.gz" + local local_checksum="$LOCAL_IMAGE_DIR/wallarm-node-6.11.0-rc1.tar.gz.sha256" + + if [ -f "$local_image" ]; then + log_message "INFO" "Found local Wallarm image: $local_image" + if verify_checksum "$local_image" "$local_checksum" "local Wallarm image"; then + log_message "INFO" "Loading Wallarm image from local directory..." + if gunzip -c "$local_image" | sudo docker load; then + log_message "SUCCESS" "Wallarm image loaded from local directory" + image_loaded=true + else + log_message "ERROR" "Failed to load Wallarm image from local directory" + fi + else + log_message "WARNING" "Local Wallarm image checksum verification failed" + fi + fi + fi + + # 3. Check current directory for compressed image (tar.gz) + if [ "$image_loaded" = "false" ]; then + log_message "INFO" "Checking current directory for Wallarm image (tar.gz)..." + local gz_image + gz_image=$(ls wallarm-node-*.tar.gz 2>/dev/null | head -1) + if [ -n "$gz_image" ]; then + log_message "INFO" "Found compressed Wallarm image: $gz_image" + # Verify checksum if .sha256 file exists + local checksum_file="${gz_image}.sha256" + if [ -f "$checksum_file" ]; then + if ! verify_checksum "$gz_image" "$checksum_file" "Wallarm image"; then + log_message "WARNING" "Wallarm image checksum verification failed, but attempting load anyway" + fi + fi + log_message "INFO" "Loading compressed Wallarm image..." + if gunzip -c "$gz_image" | sudo docker load; then + log_message "SUCCESS" "Wallarm image loaded from compressed file" + image_loaded=true + else + log_message "ERROR" "Failed to load Wallarm image from $gz_image" + fi + fi + fi + + # 4. Check current directory for uncompressed image (tar) - existing fallback + if [ "$image_loaded" = "false" ]; then + log_message "INFO" "Checking current directory for Wallarm image (tar)..." + local tar_image + tar_image=$(ls wallarm-node-*.tar 2>/dev/null | head -1) + if [ -n "$tar_image" ]; then + log_message "INFO" "Found uncompressed Wallarm image: $tar_image" + if ! sudo docker load -i "$tar_image"; then + log_message "ERROR" "Failed to load Wallarm image from $tar_image" + else + log_message "SUCCESS" "Wallarm image loaded from uncompressed file" + image_loaded=true + fi + fi + fi + + # 5. Try internal registry (if reachable per preflight check) + if [ "$image_loaded" = "false" ] && [ "$REGISTRY_REACHABLE" = "true" ]; then + log_message "INFO" "Pulling Wallarm Docker image from internal registry: $WALLARM_IMAGE_SOURCE" + if ! sudo docker pull "$WALLARM_IMAGE_SOURCE"; then + log_message "ERROR" "Failed to pull Wallarm image from internal registry" else - fail_with_remediation "No Wallarm image available" \ - "Need either: -1. Network access to $DOCKER_REGISTRY_HOST -2. Local wallarm-node-*.tar file in current directory" + # Re-tag to standard name + sudo docker tag "$WALLARM_IMAGE_SOURCE" "$WALLARM_IMAGE_TARGET" + log_message "SUCCESS" "Wallarm image pulled and tagged successfully from internal registry" + image_loaded=true + fi + fi + + # 6. Final fallback: no image available + if [ "$image_loaded" = "false" ]; then + fail_with_remediation "No Wallarm image available" \ + "Please provide a Wallarm Docker image using one of these methods: +1. GitLab (primary): Ensure network access to $GITLAB_BASE_URL +2. Local images directory: Place wallarm-node-6.11.0-rc1.tar.gz and .sha256 in $LOCAL_IMAGE_DIR/ +3. Current directory: Place wallarm-node-*.tar.gz or wallarm-node-*.tar file in current directory +4. Internal registry: Ensure network access to $DOCKER_REGISTRY_HOST + Download manually: docker pull $WALLARM_IMAGE_SOURCE + Save for offline use: docker save $WALLARM_IMAGE_TARGET -o wallarm-node-latest.tar + +Re-run the script after providing the image." + fi + + # Ensure image is tagged with standard name (for consistency) + if [ "$image_loaded" = "true" ] && [ "$REGISTRY_REACHABLE" = "false" ]; then + # If we loaded from local file, tag the loaded image with standard name + local loaded_image_id + loaded_image_id=$(sudo docker images --format "{{.ID}}" --filter "dangling=false" | head -1) + if [ -n "$loaded_image_id" ]; then + sudo docker tag "$loaded_image_id" "$WALLARM_IMAGE_TARGET" + log_message "INFO" "Tagged loaded image as $WALLARM_IMAGE_TARGET" fi fi diff --git a/wallarm-ct-uninstall.sh b/wallarm-ct-uninstall.sh old mode 100644 new mode 100755