From 3158ee7ab151cbcf6a1308254942db7422c4da61 Mon Sep 17 00:00:00 2001 From: administrator Date: Tue, 21 Apr 2026 09:34:42 +0100 Subject: [PATCH] chore: refactor git references and remove internal registry fallback - Rename GITLAB variables to GIT prefix - Update GIT_RAW_URL to use src/branch/main path - Replace 'GitLab' with 'Git Repositorys' in comments/logs - Remove internal registry/download fallback options - Update priority chains to two-tier fallback (primary + local) - Sync documentation with new terminology --- README.md | 6 +-- wallarm-ct-check.sh | 73 +++++++++++---------------- wallarm-ct-deploy.sh | 117 +++++++++++++++---------------------------- 3 files changed, 73 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index e6f1f75..7036e09 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A comprehensive solution for deploying Wallarm filtering nodes on virtual machin ## Features - **Automated Preflight Checks** – Validates system readiness, network connectivity, and resource availability -- **Smart Artifact Management** – GitLab/Forgejo-first approach with local fallback support +- **Smart Artifact Management** – Git Repositorys/Forgejo-first approach with local fallback support - **Multiple Node Support** – Deploy multiple Wallarm instances on the same VM with unique port configurations - **Interactive Configuration** – User-friendly prompts for cloud region, ports, token, and upstream applications - **Comprehensive Validation** – Network tests, port availability checks, and deployment verification @@ -170,7 +170,7 @@ sudo ./wallarm-ct-uninstall.sh The system uses a smart fallback approach for artifact retrieval: -### 1. **Primary Source**: GitLab/Forgejo Repository +### 1. **Primary Source**: Git Repositorys/Forgejo Repository - URL: `https://git.sechpoint.app/customer-engineering/wallarm` - Contains: Docker binaries and Wallarm images with SHA256 checksums - Benefits: Version control, access control, audit trail @@ -379,7 +379,7 @@ Ensure your upstream firewall/load balancer includes: ### Version Updates When updating Wallarm node version: -1. Pull new image from GitLab/Forgejo or official registry +1. Pull new image from Git Repositorys/Forgejo or official registry 2. Stop existing container 3. Deploy new container with updated image 4. Verify functionality before removing old container diff --git a/wallarm-ct-check.sh b/wallarm-ct-check.sh index 376ecd9..e978ac7 100755 --- a/wallarm-ct-check.sh +++ b/wallarm-ct-check.sh @@ -5,7 +5,7 @@ # Purpose: Validate system readiness for Wallarm deployment # Features: # - Non-interactive system validation (sudo, OS, architecture, init system) -# - Network connectivity testing (US/EU cloud, internal registry/download) +# - Network connectivity testing (US/EU cloud) # - Outputs results to .env file for deployment script # - DAU-friendly error messages with remediation # ============================================================================== @@ -54,23 +54,18 @@ else CURL_INSECURE_FLAG="" fi -# GitLab artifact URLs (primary source) - same as deployment script -GITLAB_BASE_URL="https://git.sechpoint.app/customer-engineering/wallarm" -GITLAB_RAW_URL="https://git.sechpoint.app/customer-engineering/wallarm/-/raw/main" -GITLAB_DOCKER_BINARY_URL="${GITLAB_RAW_URL}/binaries/docker-29.2.1.tgz" -GITLAB_WALLARM_IMAGE_URL="${GITLAB_RAW_URL}/images/wallarm-node-6.11.0-rc1.tar.gz" +# Git Repositorys artifact URLs (primary source) - same as deployment script +GIT_BASE_URL="https://git.sechpoint.app/customer-engineering/wallarm" +GIT_RAW_URL="https://git.sechpoint.app/customer-engineering/wallarm/src/branch/main" +GIT_DOCKER_BINARY_URL="${GIT_RAW_URL}/binaries/docker-29.2.1.tgz" +GIT_WALLARM_IMAGE_URL="${GIT_RAW_URL}/images/wallarm-node-6.11.0-rc1.tar.gz" # Local artifact directories (relative to script location) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" LOCAL_BINARY_DIR="${SCRIPT_DIR}/binaries" LOCAL_IMAGE_DIR="${SCRIPT_DIR}/images" -# Internal registry endpoints (from stealth deployment) - fallback source -INTERNAL_DOCKER_REGISTRY="https://deployment:elqXBsyT4BGXPYPeD07or8hT0Lb9Lpf@hub.ct.sechpoint.app" -INTERNAL_DOCKER_DOWNLOAD="https://deployment:elqXBsyT4BGXPYPeD07or8hT0Lb9Lpf@ct.sechpoint.app" -# Extracted hostnames (without credentials) for logging and error messages -DOCKER_REGISTRY_HOST=$(extract_hostname_from_url "$INTERNAL_DOCKER_REGISTRY") -DOCKER_DOWNLOAD_HOST=$(extract_hostname_from_url "$INTERNAL_DOCKER_DOWNLOAD") + # Cloud endpoints (from Wallarm documentation) EU_DATA_NODES=("api.wallarm.com" "node-data0.eu1.wallarm.com" "node-data1.eu1.wallarm.com") @@ -79,7 +74,7 @@ US_DATA_NODES=("us1.api.wallarm.com" "node-data0.us1.wallarm.com" "node-data1.us # Global result tracking CHECK_RESULT="pass" CHECK_ERRORS=() -GITLAB_REACHABLE="false" + GIT_REACHABLE="false" # ============================================================================== # LOGGING & ERROR HANDLING FUNCTIONS @@ -490,14 +485,14 @@ test_cloud_endpoints() { perform_network_tests() { log_message "INFO" "=== NETWORK CONNECTIVITY TESTING ===" - # Test GitLab connectivity (primary artifact source) - log_message "INFO" "Testing connectivity to GitLab artifact repository..." - GITLAB_REACHABLE="false" - if test_connectivity "$GITLAB_BASE_URL" "GitLab artifact repository"; then - GITLAB_REACHABLE="true" - log_message "SUCCESS" "GitLab artifact repository is reachable (primary source)" + # Test Git Repositorys connectivity (primary artifact source) + log_message "INFO" "Testing connectivity to Git Repositorys artifact repository..." +GIT_REACHABLE="false" + if test_connectivity "$GIT_BASE_URL" "Git Repositorys artifact repository"; then + GIT_REACHABLE="true" + log_message "SUCCESS" "Git Repositorys artifact repository is reachable (primary source)" else - log_message "WARNING" "GitLab artifact repository is not reachable - will use fallback sources" + log_message "WARNING" "Git Repositorys artifact repository is not reachable - will use fallback sources" fi # Test US cloud endpoints @@ -508,17 +503,8 @@ perform_network_tests() { local eu_reachable eu_reachable=$(test_cloud_endpoints "EU" "${EU_DATA_NODES[@]}") - # Test internal Docker registry (fallback source) local registry_reachable="false" - if test_connectivity "$INTERNAL_DOCKER_REGISTRY" "Internal Docker Registry (fallback)"; then - registry_reachable="true" - fi - - # Test internal Docker download server (fallback source) local download_reachable="false" - if test_connectivity "$INTERNAL_DOCKER_DOWNLOAD" "Internal Docker Download Server (fallback)"; then - download_reachable="true" - fi # Check for local fallback resources (multiple locations) log_message "INFO" "Checking for local artifact fallback resources..." @@ -684,13 +670,13 @@ main() { download_reachable=$(echo "$network_results" | cut -d: -f4) # Critical check: Need at least one source for Docker and Wallarm - # Priority: GitLab (primary) -> local files -> internal proxy (fallback) + # Priority: Git Repositorys (primary) -> local files - # If GitLab is reachable, we have our primary source - if [ "$GITLAB_REACHABLE" = "true" ]; then - log_message "SUCCESS" "GitLab artifact repository is reachable (primary source available)" + # If Git Repositorys is reachable, we have our primary source + if [ "$GIT_REACHABLE" = "true" ]; then + log_message "SUCCESS" "Git Repositorys artifact repository is reachable (primary source available)" else - log_message "WARNING" "GitLab artifact repository is not reachable - checking fallback sources" + log_message "WARNING" "Git Repositorys artifact repository is not reachable - checking fallback sources" # Check for local files in multiple locations local has_local_docker=false @@ -717,19 +703,19 @@ main() { # Determine if we have sufficient resources local has_sufficient_resources=true - if [ "$has_local_docker" = "false" ] && [ "$download_reachable" = "false" ]; then + if [ "$has_local_docker" = "false" ]; then log_message "ERROR" "No Docker binary source available" - log_message "ERROR" " - GitLab unreachable: $GITLAB_BASE_URL" + log_message "ERROR" " - Git Repositorys unreachable: $GIT_BASE_URL" log_message "ERROR" " - Local binaries not found in $LOCAL_BINARY_DIR/ or current directory" - log_message "ERROR" " - Internal download server unreachable: $DOCKER_DOWNLOAD_HOST" + has_sufficient_resources=false fi - if [ "$has_local_wallarm" = "false" ] && [ "$registry_reachable" = "false" ]; then + if [ "$has_local_wallarm" = "false" ]; then log_message "ERROR" "No Wallarm image source available" - log_message "ERROR" " - GitLab unreachable: $GITLAB_BASE_URL" + log_message "ERROR" " - Git Repositorys unreachable: $GIT_BASE_URL" log_message "ERROR" " - Local images not found in $LOCAL_IMAGE_DIR/ or current directory" - log_message "ERROR" " - Internal registry unreachable: $DOCKER_REGISTRY_HOST" + has_sufficient_resources=false fi @@ -737,16 +723,15 @@ main() { add_error "Insufficient resources: Need at least one source for Docker and Wallarm artifacts. Possible sources: -1. GitLab (primary): Ensure network access to $GITLAB_BASE_URL +1. Git Repositorys (primary): Ensure network access to $GIT_BASE_URL 2. Local files: Place artifacts in: - Docker binary: $LOCAL_BINARY_DIR/docker-29.2.1.tgz or current directory - - Wallarm image: $LOCAL_IMAGE_DIR/wallarm-node-6.11.0-rc1.tar.gz or current directory -3. Internal proxy: Ensure network access to $DOCKER_DOWNLOAD_HOST and $DOCKER_REGISTRY_HOST" + - Wallarm image: $LOCAL_IMAGE_DIR/wallarm-node-6.11.0-rc1.tar.gz or current directory" fi fi log_message "SUCCESS" "Network testing completed:" - log_message "SUCCESS" " GitLab Artifact Repository: $GITLAB_REACHABLE" + log_message "SUCCESS" " Git Repositorys Artifact Repository: $GIT_REACHABLE" log_message "SUCCESS" " US Cloud Reachable: $us_reachable" log_message "SUCCESS" " EU Cloud Reachable: $eu_reachable" log_message "SUCCESS" " Fallback Registry Reachable: $registry_reachable" diff --git a/wallarm-ct-deploy.sh b/wallarm-ct-deploy.sh index dc25173..6122eac 100755 --- a/wallarm-ct-deploy.sh +++ b/wallarm-ct-deploy.sh @@ -57,29 +57,20 @@ else CURL_INSECURE_FLAG="" fi -# GitLab artifact URLs (primary source) -GITLAB_BASE_URL="https://git.sechpoint.app/customer-engineering/wallarm" -GITLAB_RAW_URL="https://git.sechpoint.app/customer-engineering/wallarm/-/raw/main" -GITLAB_DOCKER_BINARY_URL="${GITLAB_RAW_URL}/binaries/docker-29.2.1.tgz" -GITLAB_DOCKER_CHECKSUM_URL="${GITLAB_RAW_URL}/binaries/docker-29.2.1.tgz.sha256" -GITLAB_WALLARM_IMAGE_URL="${GITLAB_RAW_URL}/images/wallarm-node-6.11.0-rc1.tar.gz" -GITLAB_WALLARM_CHECKSUM_URL="${GITLAB_RAW_URL}/images/wallarm-node-6.11.0-rc1.tar.gz.sha256" +# Git Repositorys artifact URLs (primary source) +GIT_BASE_URL="https://git.sechpoint.app/customer-engineering/wallarm" +GIT_RAW_URL="https://git.sechpoint.app/customer-engineering/wallarm/src/branch/main" +GIT_DOCKER_BINARY_URL="${GIT_RAW_URL}/binaries/docker-29.2.1.tgz" +GIT_DOCKER_CHECKSUM_URL="${GIT_RAW_URL}/binaries/docker-29.2.1.tgz.sha256" +GIT_WALLARM_IMAGE_URL="${GIT_RAW_URL}/images/wallarm-node-6.11.0-rc1.tar.gz" +GIT_WALLARM_CHECKSUM_URL="${GIT_RAW_URL}/images/wallarm-node-6.11.0-rc1.tar.gz.sha256" # Local artifact directories (relative to script location) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" LOCAL_BINARY_DIR="${SCRIPT_DIR}/binaries" LOCAL_IMAGE_DIR="${SCRIPT_DIR}/images" -# Internal registry endpoints (from stealth deployment) - fallback source -INTERNAL_DOCKER_REGISTRY="https://deployment:elqXBsyT4BGXPYPeD07or8hT0Lb9Lpf@hub.ct.sechpoint.app" -INTERNAL_DOCKER_DOWNLOAD="https://deployment:elqXBsyT4BGXPYPeD07or8hT0Lb9Lpf@ct.sechpoint.app" -# Extracted hostnames (without credentials) for Docker operations -DOCKER_REGISTRY_HOST=$(extract_hostname_from_url "$INTERNAL_DOCKER_REGISTRY") -DOCKER_DOWNLOAD_HOST=$(extract_hostname_from_url "$INTERNAL_DOCKER_DOWNLOAD") - DOCKER_VERSION="29.2.1" # Version from stealth deployment guide -DOCKER_STATIC_BASE_URL="${INTERNAL_DOCKER_DOWNLOAD}/linux/static/stable" -WALLARM_IMAGE_SOURCE="${DOCKER_REGISTRY_HOST}/wallarm/node:6.11.0-rc1" WALLARM_IMAGE_TARGET="wallarm/node:6.11.0-rc1" @@ -140,15 +131,15 @@ fail_with_remediation() { } # ============================================================================== -# GITLAB ARTIFACT FUNCTIONS +# GIT ARTIFACT FUNCTIONS # ============================================================================== -download_from_gitlab() { +download_from_git() { local url="$1" local output_path="$2" local description="$3" - log_message "INFO" "Attempting to download $description from GitLab..." + log_message "INFO" "Attempting to download $description from Git Repositorys..." log_message "DEBUG" "URL: $url" log_message "DEBUG" "Output path: $output_path" @@ -158,7 +149,7 @@ download_from_gitlab() { return 0 else local curl_exit=$? - log_message "ERROR" "Failed to download $description from GitLab (curl exit: $curl_exit)" + log_message "ERROR" "Failed to download $description from Git Repositorys (curl exit: $curl_exit)" # Clean up partial download if it exists if [ -f "$output_path" ]; then rm -f "$output_path" @@ -706,18 +697,18 @@ setup_docker_engine() { log_message "INFO" "Docker not found or not running. Proceeding with installation..." - # Determine binary source (priority: GitLab -> local dir -> current dir -> internal proxy) + # Determine binary source (priority: Git Repositorys -> local dir -> current dir) local binary_file="docker-$DOCKER_VERSION.tgz" local binary_path="" - # 1. Try GitLab download (primary source) - log_message "INFO" "Attempting to download Docker binary from GitLab..." - if download_from_gitlab "$GITLAB_DOCKER_BINARY_URL" "$binary_file" "Docker binary"; then - if verify_checksum "$binary_file" "$GITLAB_DOCKER_CHECKSUM_URL" "Docker binary"; then + # 1. Try Git Repositorys download (primary source) + log_message "INFO" "Attempting to download Docker binary from Git Repositorys..." + if download_from_git "$GIT_DOCKER_BINARY_URL" "$binary_file" "Docker binary"; then + if verify_checksum "$binary_file" "$GIT_DOCKER_CHECKSUM_URL" "Docker binary"; then binary_path="$binary_file" - log_message "SUCCESS" "Docker binary downloaded from GitLab and checksum verified" + log_message "SUCCESS" "Docker binary downloaded from Git Repositorys and checksum verified" else - log_message "WARNING" "GitLab Docker binary checksum verification failed, trying other sources" + log_message "WARNING" "Git Repositorys Docker binary checksum verification failed, trying other sources" # Remove corrupted download rm -f "$binary_file" fi @@ -763,30 +754,16 @@ setup_docker_engine() { fi fi - # 4. Try internal proxy (if reachable per preflight check) - if [ -z "$binary_path" ] && [ "$DOWNLOAD_REACHABLE" = "true" ]; then - # Download Docker static binary from internal server - log_message "INFO" "Downloading Docker static binary for $ARCHITECTURE from internal proxy..." - local download_url="$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz" - - if curl -fL $CURL_INSECURE_FLAG --connect-timeout 30 "$download_url" -o "$binary_file"; then - log_message "SUCCESS" "Downloaded Docker binary from internal proxy: $binary_file" - binary_path="$binary_file" - else - log_message "ERROR" "Failed to download Docker binary from $download_url" - binary_path="" - fi - fi + # 5. Final fallback: no binary available if [ -z "$binary_path" ]; then fail_with_remediation "No Docker binary available" \ "Please provide a Docker static binary using one of these methods: -1. GitLab (primary): Ensure network access to $GITLAB_BASE_URL +1. Git Repositorys (primary): Ensure network access to $GIT_BASE_URL 2. Local binaries directory: Place docker-29.2.1.tgz and .sha256 in $LOCAL_BINARY_DIR/ 3. Current directory: Place any docker-*.tgz file in current directory -4. Internal proxy: Ensure network access to $DOCKER_DOWNLOAD_HOST - Download manually: curl -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o docker.tgz + Re-run the script after providing the binary." fi @@ -877,9 +854,9 @@ Steps to fix: 1. Delete corrupted file: rm -f docker-*.tgz 2. Check disk space: df -h . 3. Try alternative sources: - a) GitLab: curl -L '$GITLAB_DOCKER_BINARY_URL' -o docker.tgz + a) Git Repositorys: curl -L '$GIT_DOCKER_BINARY_URL' -o docker.tgz b) Local directory: Check $LOCAL_BINARY_DIR/docker-29.2.1.tgz - c) Internal proxy: curl -v -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o test.tgz + 4. Verify downloaded file: file test.tgz && tar -tzf test.tgz 5. Check if tar command works: tar --version" fi @@ -945,9 +922,9 @@ Check the binary: The Docker static binary might be for wrong architecture or corrupted. Try downloading manually from one of these sources: -1. GitLab: curl -L '$GITLAB_DOCKER_BINARY_URL' -o docker.tgz +1. Git Repositorys: curl -L '$GIT_DOCKER_BINARY_URL' -o docker.tgz 2. Local directory: Check $LOCAL_BINARY_DIR/docker-29.2.1.tgz -3. Internal proxy: curl -L '$DOCKER_STATIC_BASE_URL/$ARCHITECTURE/docker-$DOCKER_VERSION.tgz' -o docker.tgz + Then extract and install: tar xzvf docker.tgz @@ -1411,28 +1388,28 @@ Check for Docker logs: deploy_wallarm_node() { log_message "INFO" "Deploying Wallarm filtering node..." - # Load Wallarm Docker image (priority: GitLab -> local dir -> current dir -> internal registry) + # Load Wallarm Docker image (priority: Git Repositorys -> local dir -> current dir) log_message "INFO" "Loading Wallarm Docker image..." local image_loaded=false - # 1. Try GitLab download (primary source) - local gitlab_image_file="wallarm-node-6.11.0-rc1.tar.gz" + # 1. Try Git Repositorys download (primary source) + local git_image_file="wallarm-node-6.11.0-rc1.tar.gz" if [ "$image_loaded" = "false" ]; then - log_message "INFO" "Attempting to download Wallarm image from GitLab..." - if download_from_gitlab "$GITLAB_WALLARM_IMAGE_URL" "$gitlab_image_file" "Wallarm Docker image"; then - if verify_checksum "$gitlab_image_file" "$GITLAB_WALLARM_CHECKSUM_URL" "Wallarm Docker image"; then - log_message "INFO" "Loading Wallarm image from GitLab download..." - if gunzip -c "$gitlab_image_file" | sudo docker load; then - log_message "SUCCESS" "Wallarm image loaded from GitLab download" + log_message "INFO" "Attempting to download Wallarm image from Git Repositorys..." + if download_from_git "$GIT_WALLARM_IMAGE_URL" "$git_image_file" "Wallarm Docker image"; then + if verify_checksum "$git_image_file" "$GIT_WALLARM_CHECKSUM_URL" "Wallarm Docker image"; then + log_message "INFO" "Loading Wallarm image from Git Repositorys download..." + if gunzip -c "$git_image_file" | sudo docker load; then + log_message "SUCCESS" "Wallarm image loaded from Git Repositorys download" image_loaded=true else - log_message "ERROR" "Failed to load Wallarm image from GitLab download" + log_message "ERROR" "Failed to load Wallarm image from Git Repositorys download" fi # Cleanup downloaded file - rm -f "$gitlab_image_file" + rm -f "$git_image_file" else - log_message "WARNING" "GitLab Wallarm image checksum verification failed" - rm -f "$gitlab_image_file" + log_message "WARNING" "Git Repositorys Wallarm image checksum verification failed" + rm -f "$git_image_file" fi fi fi @@ -1499,28 +1476,16 @@ deploy_wallarm_node() { fi fi - # 5. Try internal registry (if reachable per preflight check) - if [ "$image_loaded" = "false" ] && [ "$REGISTRY_REACHABLE" = "true" ]; then - log_message "INFO" "Pulling Wallarm Docker image from internal registry: $WALLARM_IMAGE_SOURCE" - if ! sudo docker pull "$WALLARM_IMAGE_SOURCE"; then - log_message "ERROR" "Failed to pull Wallarm image from internal registry" - else - # Re-tag to standard name - sudo docker tag "$WALLARM_IMAGE_SOURCE" "$WALLARM_IMAGE_TARGET" - log_message "SUCCESS" "Wallarm image pulled and tagged successfully from internal registry" - image_loaded=true - fi - fi + # 6. Final fallback: no image available if [ "$image_loaded" = "false" ]; then fail_with_remediation "No Wallarm image available" \ "Please provide a Wallarm Docker image using one of these methods: -1. GitLab (primary): Ensure network access to $GITLAB_BASE_URL +1. Git Repositorys (primary): Ensure network access to $GIT_BASE_URL 2. Local images directory: Place wallarm-node-6.11.0-rc1.tar.gz and .sha256 in $LOCAL_IMAGE_DIR/ 3. Current directory: Place wallarm-node-*.tar.gz or wallarm-node-*.tar file in current directory -4. Internal registry: Ensure network access to $DOCKER_REGISTRY_HOST - Download manually: docker pull $WALLARM_IMAGE_SOURCE + Save for offline use: docker save $WALLARM_IMAGE_TARGET -o wallarm-node-latest.tar Re-run the script after providing the image."