#!/usr/bin/env bash set -eo pipefail PROJECT_NAME="crowd" DOCKER_NETWORK_SUBNET="${CROWD_NETWORK_SUBNET:-10.90.0.0/24}" DOCKER_NETWORK_GATEWAY="${CROWD_NETWORK_GATEWAY:-10.90.0.1}" DOCKET_TEST_NETWORK_SUBNET="${CROWD_TEST_NETWORK_SUBNET:-10.91.0.0/24}" DOCKER_TEST_NETWORK_GATEWAY="${CROWD_TEST_NETWORK_GATEWAY:-10.91.0.1}" CLI_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $CLI_HOME/utils CONTAINERS=`ls -p $CLI_HOME/services | grep -v / | sed 's/\.[^.]*$//'` BUILDERS=`ls -p $CLI_HOME/builders | grep -v / | sed 's/\.[^.]*$//'` _DC="docker compose" arch=$(uname -m) if [ "$arch" == 'arm64' ]; then DOCKER_PLATFORM_FLAGS="--platform=linux/arm64/v8" else DOCKER_PLATFORM_FLAGS="--platform=linux/amd64" fi function prepare_dev_env() { if [[ ! -f "$CLI_HOME/../backend/.env.override.local" ]]; then echo "# Here you can put environment variables that you would like to override when using local environment" > "$CLI_HOME/../backend/.env.override.local" fi if [[ ! -f "$CLI_HOME/../backend/.env.override.composed" ]]; then echo "# Here you can put environment variables that you would like to override when using local environment" > "$CLI_HOME/../backend/.env.override.composed" fi if [[ ! -f "$CLI_HOME/../frontend/.env.override.local" ]]; then echo "# Here you can put environment variables that you would like to override when using local environment" > "$CLI_HOME/../frontend/.env.override.local" fi if [[ ! -f "$CLI_HOME/../frontend/.env.override.composed" ]]; then echo "# Here you can put environment variables that you would like to override when using local environment" > "$CLI_HOME/../frontend/.env.override.composed" fi set +eo pipefail $_DC >> /dev/null 2>&1 if [[ ! $? -eq 0 ]]; then _DC="docker-compose" $_DC >> /dev/null 2>&1 if [[ ! $? -eq 0 ]]; then error "Docker compose not detected!" exit 1 fi fi set -eo pipefail } prepare_dev_env function prepare_service_string() { if [ $1 == "HELP" ]; then delimiter1=" | "; delimiter2=" => | "; fi if [ $1 == "LIST" ]; then delimiter1="\|"; delimiter2="\|"; fi for cmd in $SERVICE_CMD; do string+="$cmd$delimiter1"; done for container in $2; do string+="$container$delimiter2"; done echo ${string%??} } HELP_STRING=$(prepare_service_string HELP "$CONTAINERS") LIST_STRING=$(prepare_service_string LIST "$CONTAINERS") HELP_BUILD_STRING=$(prepare_service_string HELP "$BUILDERS") function build() { HELP="${RESET}\nUsage:\n ./cli build [ $HELP_BUILD_STRING ]\n" [[ -z "$1" ]] && say "$HELP" && exit 1 if [[ $BUILDERS =~ (^|[[:space:]])"$1"($|[[:space:]]) ]] ; then build_and_publish "$@" else error "Invalid command '$1'" && say "$HELP" exit 1; fi } function create_migration() { MIG_NAME="$1" MIG_VERSION=$(date +%s) UP_MIG_FILE="${CLI_HOME}/../backend/src/database/migrations/V${MIG_VERSION}__${MIG_NAME}.sql" DOWN_MIG_FILE="${CLI_HOME}/../backend/src/database/migrations/U${MIG_VERSION}__${MIG_NAME}.sql" touch $UP_MIG_FILE touch $DOWN_MIG_FILE yell "Created ${MIG_FILE}"; } function build_and_publish() { VERSION="$2" if [[ -z "${VERSION}" ]]; then COMMIT_HASH=`git rev-parse --short HEAD` TS_VERSION=$(date +%s) VERSION="$TS_VERSION.$COMMIT_HASH" fi source $CLI_HOME/builders/$1.env if [[ ${BUILD} ]]; then say "Building $REPO version $VERSION with dockerfile '$DOCKERFILE' and context '$CONTEXT' and pushing it to $REPO:$VERSION" docker build --platform linux/amd64 --tag "$REPO:$VERSION" -f "$DOCKERFILE" "$CONTEXT" fi if [[ ${PUSH} ]]; then say "Pushing image $REPO version $VERSION to $REPO:$VERSION" docker push "$REPO:$VERSION" fi } function db_backup() { [[ -z "$1" ]] && error "Dump name has to be provided as first parameter!" && exit 1 mkdir -p $CLI_HOME/db_dumps say "Cloning local database to $CLI_HOME/db_dumps/$1.dump!" docker exec -t ${PROJECT_NAME}_db_1 bash -c "PGPASSWORD=example pg_dump -F c -d crowd-web -U postgres > /$1.dump" docker cp ${PROJECT_NAME}_db_1:/$1.dump $CLI_HOME/db_dumps/$1.dump say "All done!" } function restore_db_backup() { [[ -z "$1" ]] && error "Dump name has to be provided as first parameter!" && exit 1 say "First we need to clean up the scaffold!" scaffold_destroy up_scaffold say "Sleeping for 5 seconds for the database container to start up!" sleep 5 say "Restoring dump from $CLI_HOME/db_dumps/$1.dump" docker cp $CLI_HOME/db_dumps/$1.dump ${PROJECT_NAME}_db_1:/backup.dump # docker exec -t ${PROJECT_NAME}_db_1 bash -c "PGPASSWORD=example dropdb -U postgres crowd-web && PGPASSWORD=example createdb -U postgres crowd-web" set +eo pipefail docker exec -t ${PROJECT_NAME}_db_1 bash -c "PGPASSWORD=example pg_restore --clean -U postgres -d crowd-web backup.dump && rm -f backup.dump" set -eo pipefail post_up_scaffold say "All done!" } function scaffold() { HELP="${RESET}\nUsage:\n ./cli scaffold [ up | down | destroy | reset ]\n" [[ -z "$1" ]] && say "$HELP" && exit 1 while test $# -gt 0 do case "$1" in up) up_scaffold post_up_scaffold exit; ;; down) down_scaffold exit; ;; destroy) scaffold_destroy exit; ;; reset) scaffold_reset exit; ;; create-migration) create_migration $2 exit; ;; migrate-up) migrate_local exit; ;; up-test) up_test_scaffold exit; ;; *) error "Invalid command '$1'" && say "$HELP" exit 1; ;; esac shift done } function service() { HELP="${RESET}\nUsage:\n ./cli service [ $HELP_STRING ]\n" [[ -z "$1" ]] && say "$HELP" && exit 1 while test $# -gt 0 do case "$1" in list) docker container ls | grep $LIST_STRING exit; ;; up-all) start_all_containers exit; ;; *) if [[ $CONTAINERS =~ (^|[[:space:]])"$1"($|[[:space:]]) ]] ; then service_manipulator $1 $2 $3 exit; else error "Invalid command '$1'" && say "$HELP" exit 1; fi ;; esac shift done } function service_manipulator() { HELP="${RESET}\nUsage:\n ./cli service $1 [ up | down | restart | status | logs | id ]\n" [[ -z "$2" ]] && say "$HELP" && exit 1 while test $# -gt 0 do case "$2" in up) start_service "$1" exit; ;; down) kill_containers "$1" exit; ;; restart) kill_containers "$1" && start_service "$1" exit; ;; status) print_container_status "$1" exit; ;; logs) get_logs "$1" exit; ;; id) get_container_id "$1" exit; ;; *) error "Invalid command '$2'" && say "$HELP" exit 1; ;; esac shift done } function start_service() { export USER_ID=$(id -u) export GROUP_ID=$(id -g) if [[ ${DEV} ]]; then $_DC --compatibility -p $PROJECT_NAME -f "$CLI_HOME/services/${1}.yaml" up --build -d ${1}-dev else $_DC --compatibility -p $PROJECT_NAME -f "$CLI_HOME/services/${1}.yaml" up --build -d ${1} fi } function kill_containers() { $_DC --compatibility -p $PROJECT_NAME -f "$CLI_HOME/services/${1}.yaml" rm -fs ${1} ${1}-dev } function get_logs() { if [[ ${DEV} ]]; then docker container logs -f $(get_container_id "$1-dev") else docker container logs -f $(get_container_id "$1") fi } function print_container_status() { CONTAINER_STATUS=$(check_container_status $1) if [[ ${CONTAINER_STATUS} ]]; then check_container_status "$1" else error "Down." exit 1; fi } function get_container_id() { docker container ls -a | grep "${PROJECT_NAME}_${1}_" | tr " " "\n" | head -n 1 } function check_container_status() { docker container ls -a | grep "${PROJECT_NAME}_${1}_" } # function scaffold_set_up_network() { NETWORK_NAME="$1" NETWORK_SUBNET="$2" NETWORK_GATEWAY="$3" set +e pipefail NETWORK_ID=$(docker network ls | grep -F -e "$NETWORK_NAME " | tr " " "\n" | head -n 1) set -e pipefail if [[ ${NETWORK_ID} ]]; then say "The $NETWORK_NAME network is up and running." else docker network create -d bridge --subnet "$NETWORK_SUBNET" --gateway "$NETWORK_GATEWAY" "$NETWORK_NAME" fi } function migrate_local() { say "Building flyway migration image..." docker build $DOCKER_PLATFORM_FLAGS -t crowd_flyway -f $CLI_HOME/../backend/src/database/Dockerfile.flyway $CLI_HOME/../backend/src/database --load say "Applying database migrations!" docker run --rm --network "${PROJECT_NAME}-bridge" \ -e PGHOST=db \ -e PGPORT=5432 \ -e PGUSER=postgres \ -e PGPASSWORD=example \ -e PGDATABASE=crowd-web \ crowd_flyway } function up_test_scaffold() { scaffold_set_up_network "${PROJECT_NAME}-bridge-test" $DOCKET_TEST_NETWORK_SUBNET $DOCKER_TEST_NETWORK_GATEWAY $_DC -p "$PROJECT_NAME-test" -f $CLI_HOME/../backend/docker-compose.test.yaml down $_DC -p "$PROJECT_NAME-test" -f $CLI_HOME/../backend/docker-compose.test.yaml up -d migrate_test } function migrate_test() { say "Building flyway migration image..." docker build -t crowd_flyway -f $CLI_HOME/../backend/src/database/Dockerfile.flyway $CLI_HOME/../backend/src/database say "Applying database migrations!" docker run --rm --network "${PROJECT_NAME}-bridge-test" \ -e PGHOST=db-test \ -e PGPORT=5432 \ -e PGUSER=postgres \ -e PGPASSWORD=example \ -e PGDATABASE=crowd-web \ crowd_flyway } function source_edition() { __CROWD_EDITION=$(source $CLI_HOME/../backend/.env.dist.local && source $CLI_HOME/../backend/.env.override.local && echo $CROWD_EDITION) nl say "Crowd edition detected: $__CROWD_EDITION" } function check_init_premium() { source_edition if [ $__CROWD_EDITION == "crowd-hosted" ]; then $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/premium-scaffold.yaml up -d --build init_unleash fi } function install_libs() { (cd $CLI_HOME/.. && pnpm i --frozen-lockfile) } function init_unleash() { install_libs (cd $CLI_HOME/../backend && source source-local.sh && pnpm run script:unleash-init) } function up_scaffold() { scaffold_set_up_network "$PROJECT_NAME-bridge" $DOCKER_NETWORK_SUBNET $DOCKER_NETWORK_GATEWAY $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml up -d --build } function post_up_scaffold() { migrate_local bash $CLI_HOME/nango-integrations.sh check_init_premium } function down_scaffold() { $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml down $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/premium-scaffold.yaml down } function scaffold_destroy() { say "\nWill delete all local crowd state data (docker volumes) if you have any. Are you sure?" select reset_system_condition in "Yes" "No"; do case $reset_system_condition in 'Yes' ) scaffold_destroy_confirmed; break;; 'No' ) yell "Canceled!"; break;; esac done } function scaffold_reset() { scaffold_destroy up_scaffold post_up_scaffold } function kill_all_containers() { for i in $CONTAINERS do say "Killing service $i." if [[ $(check_container_status ${i}) ]]; then docker rm -f $(get_container_id ${i}) yell "Service $i killed." elif [[ $(check_container_status ${i}-dev) ]]; then docker rm -f $(get_container_id ${i}-dev) yell "Service $i-dev killed." else error "Service $i not running." fi nl done } function scaffold_destroy_confirmed() { kill_all_containers $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml down $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/premium-scaffold.yaml down # needed because if there are no volumes this might cause the script to exit set +eo pipefail VOLUMES=$(docker volume ls | tail -n +2 | tr -s " " | cut -d' ' -f2 | grep $PROJECT_NAME) set -eo pipefail if [[ ${VOLUMES} ]]; then _IFS=$IFS IFS=$' ' NAMES=$VOLUMES IFS=$_IFS for name in $NAMES do say "Destroying volume $name!" docker volume rm -f $name done fi } function wait_for_db() { say "Waiting for scaffold to start!" sleep 3 while [[ ! $(docker container ls | grep $PROJECT_NAME | grep db | grep Up) ]]; do sleep 1 done say "Scaffold is up and running!" } function start() { if [[ -z "$CLEAN_START" ]]; then up_scaffold post_up_scaffold else scaffold_reset fi for SERVICE in $CONTAINERS do if [[ ${#INGORED_SERVICES[@]} -ne 0 ]]; then for IGNORED_SERVICE in "${INGORED_SERVICES[@]}" do if [[ "$SERVICE" == "${IGNORED_SERVICE}" ]]; then SKIP=1 break fi done fi if [[ -z "$SKIP" ]]; then say "Starting service $SERVICE." start_service $SERVICE nl fi unset SKIP done } SCRIPT_USAGE="${YELLOW}${PROJECT_NAME} CLI ${RESET}\n\nExample usage: ./cli [ start, start-dev, clean-start, clean-start-dev, start-backend, start-backend-dev, clean-start-backend, clean-start-backend-dev, start-premium, start-premium-dev, clean-start-premium, clean-start-premium-dev, scaffold =>, service =>, build =>, build-and-push ]" [[ -z "$1" ]] && say "$SCRIPT_USAGE" && exit 1 while test $# -gt 0 do case "$1" in scaffold) scaffold $2 $3 $4 $5 exit; ;; service) service $2 $3 $4 $5 exit; ;; services) service $2 $3 $4 $5 exit; ;; start) start exit; ;; start-e2e) declare -a INGORED_SERVICES=("python-worker" "job-generator" "discord-ws" "data-sink-worker" "integration-data-worker" "integration-run-worker" "integration-stream-worker") start exit; ;; start-be) declare -a INGORED_SERVICES=("python-worker" "frontend" "discord-ws") start exit; ;; start-dev) DEV=1 start exit; ;; clean-start) CLEAN_START=1 start exit; ;; clean-start-dev) CLEAN_START=1 DEV=1 start exit; ;; clean-start-backend) declare -a INGORED_SERVICES=("frontend") CLEAN_START=1 DEV=1 start exit; ;; start-backend) declare -a INGORED_SERVICES=("frontend") start exit; ;; start-backend-dev) declare -a INGORED_SERVICES=("frontend") DEV=1 start exit; ;; clean-start-backend) declare -a INGORED_SERVICES=("frontend") CLEAN_START=1 start exit; ;; clean-start-backend-dev) declare -a INGORED_SERVICES=("frontend") CLEAN_START=1 DEV=1 start exit; ;; start-premium) start exit; ;; start-premium-dev) DEV=1 start exit; ;; clean-start-premium) CLEAN_START=1 start exit; ;; clean-start-premium-dev) CLEAN_START=1 DEV=1 start exit; ;; start-premium) start exit; ;; start-premium-dev) DEV=1 exit; ;; clean-start-premium) CLEAN_START=1 exit; ;; clean-start-premium-dev) CLEAN_START=1 DEV=1 exit; ;; build) BUILD=1 build $2 $3 exit; ;; build-and-push) BUILD=1 PUSH=1 build $2 $3 exit; ;; push) PUSH=1 build $2 $3 exit; ;; db-backup) db_backup $2 exit; ;; db-restore) restore_db_backup $2 exit; ;; *) error "Invalid command '$1'" && say "$SCRIPT_USAGE" exit 1; ;; esac shift done