Install Guide

Setup the Project Directory

mkdir -p /home/websites/atlas/hub
cd /home/websites/atlas/hub

Run the Installer

curl -sSL https://atlas.bi/installers/hub.sh | bash -

#!/usr/bin/env bash
# Setup global variables.
APP=atlas-hub
SOURCE=https://api.github.com/repos/atlas-bi/Hub/tarball
# load config
NGINX_FILE="$APP.conf"
PM2_PREFIX="$APP"
if [[ -f "installer.conf" ]]; then
. installer.conf
fi
# Setup basic colors
color() {
  YELLOW=$(printf '\033[1m\033[33m')
  BLUE=$(printf '\033[1m\033[34m')
  RED=$(printf '\033[1m\033[31m')
  RESET=$(printf '\033[0m') # No Color
  GREEN=$(printf '\033[1m\033[32m')
  CYAN=$(printf '\033[1m\033[36m')
  BOLD=$(printf '\033[1m')
}

color

fmt_yellow() {
  echo "${YELLOW}$1${RESET}"
}
fmt_red() {
  echo "${RED}$1${RESET}"
}
fmt_blue() {
  echo "${BLUE}$1${RESET}"
}
fmt_green() {
  echo "${GREEN}$1${RESET}"
}
fmt_cyan() {
  echo "${BLUE}$1${RESET}"
}
check_command() {
  if ! [ -x "$(command -v $1)" ]; then
    fmt_red "Error: $1 is not installed. ${GREEN}See https://atlas.bi/docs/" >&2
    exit 1
  fi
}

warn_command() {
  if ! [ -x "$(command -v $1)" ]; then
    fmt_cyan "$2" >&2
  fi
}

check_file() {
  if ! [[ -n $(compgen -G $1) ]]; then
    fmt_red "File $1 must be created before running this script. ${GREEN}See https://atlas.bi/docs/" >&2
    exit 1
  fi
}

exporter() {
  echo $1 | tee -a .env .env.local >/dev/null
}
random_number() {
  floor=3000
  range=3999
  number=0
  while [ "$number" -le $floor ]
  do
    number=$RANDOM
    let "number %= $range"
  done
  echo $number
}

get_port() {
  PORT=$(random_number)
  while [[ $(lsof -i -P -n | grep :$PORT) ]]
  do
    PORT=$(random_number)
  done
  echo $PORT
}
# from https://dev.to/justincy/blue-green-node-js-deploys-with-nginx-bkc
nginx_workers() {
  echo $(ps -ef | grep "nginx: worker process" | grep -v grep | wc -l)
}

nginx_reload() {
  numWorkerProcesses=$(nginx_workers)
  nginx -s reload

  # Wait for the old nginx workers to be retired before we kill the old server.
  while [ $(nginx_workers) -ne $numWorkerProcesses ]
  do
    sleep 1;
  done;
}
set -Eeuo pipefail
trap cleanup SIGINT SIGTERM ERR EXIT

configure(){
    check_file config_cust.py

    fmt_yellow "Update config_cust.py file in site.."

    pm2 list | grep -oP "$PM2_PREFIX-((runner|scheduler)-)?\d+" | uniq | grep -oP "\d+" | uniq  | while IFS=$'\n' read DIRECTORY; do
      if [ -d "$DIRECTORY" ]; then

        # get old ports
        SCHEDULER=$(cat "$DIRECTORY/config_cust.py" | grep -oE "SCHEDULER_HOST.*?" | sed 's/\//\\\//g')
        RUNNER=$(cat "$DIRECTORY/config_cust.py" | grep -oE "RUNNER_HOST.*?" | sed 's/\//\\\//g')

        cp config_cust.py $DIRECTORY
        sed -i "s/SCHEDULER_HOST.*/${SCHEDULER}/g" "$DIRECTORY/config_cust.py"
        sed -i "s/RUNNER_HOST.*/${RUNNER}/g" "$DIRECTORY/config_cust.py"
      fi
    done

    fmt_yellow "Restarting processes.."
    pm2 list | grep -oP "$PM2_PREFIX-((runner|scheduler)-)?\d+" | uniq | while IFS=$'\n' read process; do
      pm2 restart $process
    done
}

usage() {
  cat << EOF

${BOLD}Usage: $(basename "${BASH_SOURCE[0]}") [-h, -b, -c, -u]

${BLUE}Atlas Hub Installer.${RESET}

Available options:

    -h, --help               Print this help and exit
    -c, --configure          Reconfigure Atlas Hub
    -i, --install [DEFAULT]  Install or Upgrade Atlas Hub

Additional Altas Hub Help at https://atlas.bi/docs/hub

EOF
  exit
}

install() {
  # Check if commands and files exist.
check_command node
check_command npm
check_command curl
check_command pm2
check_command nginx
check_command lsof
check_command grep
check_command poetry
check_file config_cust.py
check_file "/etc/nginx/**/$NGINX_FILE"

# Get free internal ports.
fmt_yellow "Finding free ports.."

PORT=$(get_port)
RUNNER_PORT=$(get_port)
SCHEDULER_PORT=$(get_port)

fmt_blue "Using web port $PORT"
fmt_blue "Using runner port $RUNNER_PORT"
fmt_blue "Using scheduler port $SCHEDULER_PORT"

# Download the latest release.
fmt_yellow "Downloading latest version into $(pwd)/$PORT.."

mkdir "$PORT"
curl -sSL "$SOURCE" | tar zxf - -C "$PORT" --strip-components=1
cd "$PORT"

fmt_blue "Downloaded version $(npm pkg get version | tr -d '"')"

# Copy in the .env file.
fmt_yellow "Setting up configuration.."
cp ../config_cust.py .

# update ports
sed -i "s/SCHEDULER_HOST.*/SCHEDULER_HOST = \"http:\/\/127.0.0.1:${SCHEDULER_PORT}\/api\"/g" config_cust.py
sed -i "s/RUNNER_HOST.*/RUNNER_HOST = \"http:\/\/127.0.0.1:${RUNNER_PORT}\/api\"/g" config_cust.py

# make log dirs
mkdir -p logs


fmt_yellow "Installing python packages.."
poetry config --local virtualenvs.in-project true
poetry config --local virtualenvs.create true
poetry install --only main

export FLASK_ENV=production
export FLASK_DEBUG=0

fmt_yellow "Installing node packages and building static resources.."
npm install --loglevel error --no-fund --no-audit
.venv/bin/flask --app=web assets build


# Apply database migrations.
fmt_yellow "Applying database migrations.."
cp web/model.py runner/model.py
cp web/model.py scheduler/model.py

.venv/bin/flask --app=web db upgrade
.venv/bin/flask --app=web cli seed

# Set a few process names.
APP_PROCESS="$PM2_PREFIX-$PORT"
RUNNER_PROCESS="$PM2_PREFIX-runner-$RUNNER_PORT"
SCHEDULER_PROCESS="$PM2_PREFIX-scheduler-$SCHEDULER_PORT"


fmt_yellow "Starting new services.."

APP_CMD=".venv/bin/gunicorn --worker-class=gevent --workers 3 --threads 30 --timeout 999999999 --access-logfile $(pwd)/logs/access.log --error-logfile $(pwd)/logs/error.log --capture-output --bind 0.0.0.0:$PORT --umask 007 web:app"
SCHEDULER_CMD=".venv/bin/gunicorn --worker-class=gevent --workers 1 --threads 30 --timeout 999999999 --access-logfile $(pwd)/logs/access.log --error-logfile $(pwd)/logs/error.log --capture-output --bind  0.0.0.0:$SCHEDULER_PORT --umask 007 scheduler:app"
RUNNER_CMD=".venv/bin/gunicorn --worker-class=gevent --worker-connections=1000 --workers $(nproc --all) --threads 30 --timeout 999999999 --access-logfile $(pwd)/logs/access.log --error-logfile $(pwd)/logs/error.log --capture-output --bind 0.0.0.0:$RUNNER_PORT --umask 007 runner:app"

echo $APP_CMD

pm2 start "$APP_CMD" --name="$APP_PROCESS"
pm2 start "$SCHEDULER_CMD" --name="$SCHEDULER_PROCESS"
pm2 start "$RUNNER_CMD" --name="$RUNNER_PROCESS"

fmt_blue "Done setting up."

cd ..

fmt_yellow "Updating nginx.."
sed -i "s/localhost:3[0-9]*/localhost:${PORT}/" `find -L /etc/nginx -name "$NGINX_FILE"`

fmt_yellow "Gracefully reloading nginx..."
nginx_reload

fmt_yellow "Removing old pm2 processes.."

# gnu grep
pm2 list | grep -oP "$PM2_PREFIX-((runner|scheduler)-)?\d+" | uniq | while IFS=$'\n' read process; do
  if [[ $process != $APP_PROCESS && $process != $RUNNER_PROCESS && $process != $SCHEDULER_PROCESS ]];
  then
    fmt_yellow "Removing $process"
    pm2 delete $process || true
  fi
done

pm2 save

fmt_yellow "Archiving old installs.."

for olddir in $(ls -d 3*); do
  if [[ $olddir != $PORT ]];
  then
    fmt_yellow "Moving $olddir"
    mv -f $olddir "backup-$olddir"
  fi
done;

fmt_blue "Finished cleaning up."

fmt_green "Thanks for installing Atlas Hub!"
echo ""
fmt_green "Read the full install guide at https://atlas.bi/docs/hub/"
echo ""
fmt_blue "Next Steps"
echo ""
fmt_cyan "Remove Backups"
echo ""
echo ${YELLOW}Back folders can be manually removed. ${BLUE}rm -r $(pwd)/backup-*
echo ""

cat <<EOF
${CYAN}Current Configuration

${YELLOW}Web process was started with ${BLUE}pm2 start "$APP_CMD" --name="$APP_PROCESS"
${YELLOW}Scheduler process was started with ${BLUE}pm2 start "$SCHEDULER_CMD" --name="$SCHEDULER_PROCESS"
${YELLOW}Runner process was started with ${BLUE}pm2 start "$RUNNER_CMD" --name="$RUNNER_PROCESS"

${CYAN}Updating App Settings

${YELLOW}1. Update user configuration file ${BLUE}nano $(pwd)/config_cust.py
${YELLOW}2. Reconfigure with ${BLUE}curl -sSL https://atlas.bi/installers/hub.sh | bash -s -- --configure

${CYAN}Updating Nginx Settings

${YELLOW}1. Update configuration file ${BLUE}nano $(find -L /etc/nginx -name "$NGINX_FILE")
${YELLOW}2. Reload nginx ${BLUE}nginx -s reload

${CYAN}Monitoring and Viewing Logs

${YELLOW}Live Logging ${BLUE}pm2 monit
${YELLOW}Log files can be viewed in the $(pwd)/$PORT/logs folder.

${RESET}
EOF

warn_command ufw "Recommendation: secure your server with ufw!"
echo ""

}

cleanup() {
  trap - SIGINT SIGTERM ERR EXIT
}

die() {
  echo >&2 -e "${1-}"
  exit 1
}

# https://betterdev.blog/minimal-safe-bash-script-template/
parse_params() {
  while :; do
    case "${1-}" in
    -h | --help) usage;break ;;

    -c | --configure) configure;break ;;
    -i | --install) install;break ;;
    -?*) die "${RED}Unknown option: $1. Run $(basename "${BASH_SOURCE[0]}") -h for help.${RESET}";break ;;
    *)  install;break ;;
    esac
    shift
  done

  return 0
}

parse_params "$@"

Advanced Installer Configuration

Some items in the installer can be configured through an installer.conf file in the install directory. These settings are helpful if you are running multiple instances of Atlas Hub on the same server.

Key Definition
NGINX_FILE Alternate name for the nginx config file. Default is atlas-hub.conf.
PM2_PREFIX Alternate prefix for pm2 processes. Default is atlas-hub.

The file format should be:

KEY=VALUE

Next Steps

Nice job installing! You can access the website with a default username of “admin” to try things out. Now it is time to fully configure the app. See the configuration guide for a complete list of options.

Tips

Making Connections to SQL Server Databases

If you will be using SQL Server databases as a data source you will need to install the ODBC package from Microsoft.

Making Connections to a LAN

If you use host names vs IP addresses in your config files be sure to update hosts file nano /etc/hosts to include the IP address of any internal domain hosts you will use. For example, LDAP server, GIT server, any databases you plan to query, etc.

Authentication

There are two primary authentication options -

  • SAML2
  • LDAP

SAML2

The PySAML2 library is used for SAML authentication, and all the sp configuration parameters are supported. The default config file includes an ADFS setup example.

Saml metadata is accessible at /saml2/metadata/.

LDAP

LDAP login follows this basic process:

  1. config_cust.py file holds the general connection info. A connection to the ldap server is made with the service account credentials supplied in the config file.
  2. Once a connection is established and a user attempts to access the site the package first verifies that the user exists, by doing a search for the user. If the user exists we save their details and groups.
  3. If the user exists then we attempt to log them in… this returns true if they had a valid username/pass.
  4. Finally, as this site can be restricted to users in a certain LDAP group, for example, we only allow users that have the “Analytics” group on their profile.