Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Add password based auth for postgres #37068

Open
wants to merge 16 commits into
base: release
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions deploy/docker/fs/opt/appsmith/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,12 @@ init_postgres() {
tlog "Initializing local Postgres data folder"
su postgres -c "env PATH='$PATH' initdb -D $POSTGRES_DB_PATH"
fi
cp /opt/appsmith/postgres/appsmith_hba.conf "$POSTGRES_DB_PATH/pg_hba.conf"
abhvsn marked this conversation as resolved.
Show resolved Hide resolved
# PostgreSQL requires strict file permissions for the pg_hba.conf file. Add file permission settings after copying the configuration file.
# 600 is the recommended permission for pg_hba.conf file for read and write access to the owner only.
chown postgres:postgres "$POSTGRES_DB_PATH/pg_hba.conf"
chmod 600 "$POSTGRES_DB_PATH/pg_hba.conf"

create_appsmith_pg_db "$POSTGRES_DB_PATH"
else
runEmbeddedPostgres=0
Expand Down
34 changes: 18 additions & 16 deletions deploy/docker/fs/opt/appsmith/pg-utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ DB_HOST="127.0.0.1"
DB_PORT="5432"
DB_SCHEMA="appsmith"
DB_NAME="appsmith"
postgres_admin_user="postgres"
POSTGRES_ADMIN_USER="postgres"

waitForPostgresAvailability() {
if [ -z "$PG_DB_HOST" ]; then
Expand Down Expand Up @@ -107,30 +107,32 @@ init_pg_db() {
if [[ "$PG_DB_HOST" == "localhost" || "$PG_DB_HOST" == "127.0.0.1" ]]; then

# Check if the database exists
DB_CHECK=$(psql -h "$PG_DB_HOST" -p "$PG_DB_PORT" -U postgres -d "postgres" -tAc "SELECT 1 FROM pg_database WHERE datname='$PG_DB_NAME'")
DB_CHECK=$(psql -p "$PG_DB_PORT" -U "$POSTGRES_ADMIN_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$PG_DB_NAME'")

if [ "$DB_CHECK" != "1" ]; then
echo "Database $PG_DB_NAME does not exist. Creating database..."
psql -h "$PG_DB_HOST" -p "$PG_DB_PORT" -U postgres -d "postgres" -c "CREATE DATABASE $PG_DB_NAME;"
psql -p "$PG_DB_PORT" -U "$POSTGRES_ADMIN_USER" -c "CREATE DATABASE $PG_DB_NAME;"
else
echo "Database $PG_DB_NAME already exists."
fi

# Check if the schema exists
SCHEMA_CHECK=$(psql -h "$PG_DB_HOST" -p "$PG_DB_PORT" -U postgres -d "$PG_DB_NAME" -tAc "SELECT 1 FROM information_schema.schemata WHERE schema_name='appsmith'")
SCHEMA_CHECK=$(psql -p "$PG_DB_PORT" -U "$POSTGRES_ADMIN_USER" -d "$PG_DB_NAME" -tAc "SELECT 1 FROM information_schema.schemata WHERE schema_name='appsmith'")

# Create schema and user if not exists
if [ "$SCHEMA_CHECK" != "1" ]; then
echo "Creating user '$PG_DB_USER' with password "
psql -h "$PG_DB_HOST" -p "$PG_DB_PORT" -U postgres -d "$PG_DB_NAME" -c "CREATE USER \"$PG_DB_USER\" WITH PASSWORD '$PG_DB_PASSWORD';"
psql -p "$PG_DB_PORT" -U "$POSTGRES_ADMIN_USER" -d "$PG_DB_NAME" -c "CREATE USER \"$PG_DB_USER\" WITH PASSWORD '$PG_DB_PASSWORD';"

echo "Schema 'appsmith' does not exist. Creating schema..."
psql -h "$PG_DB_HOST" -p "$PG_DB_PORT" -U postgres -d "$PG_DB_NAME" -c "CREATE SCHEMA appsmith;"
psql -p "$PG_DB_PORT" -U "$POSTGRES_ADMIN_USER" -d "$PG_DB_NAME" -c "CREATE SCHEMA appsmith;"
fi
USER=$PG_DB_USER SCHEMA="appsmith" DB=$PG_DB_NAME HOST=$PG_DB_HOST PORT=$PG_DB_PORT grant_permissions_for_schema

echo "Creating pg_trgm extension..."
psql -h "$PG_DB_HOST" -p "$PG_DB_PORT" -U postgres -d "$PG_DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
psql -p "$PG_DB_PORT" -U "$POSTGRES_ADMIN_USER" -d "$PG_DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;"

abhvsn marked this conversation as resolved.
Show resolved Hide resolved
# Grant permissions to the user on the schema
USER=$PG_DB_USER SCHEMA="appsmith" DB=$PG_DB_NAME HOST=$PG_DB_HOST PORT=$PG_DB_PORT grant_permissions_for_local_db_schema

else
echo "Remote PostgreSQL detected, running as current user."
PGPASSWORD=$PG_DB_PASSWORD psql -h "$PG_DB_HOST" -p "$PG_DB_PORT" -U "$PG_DB_USER" -d "$PG_DB_NAME" -c "CREATE SCHEMA IF NOT EXISTS appsmith;"
Expand Down Expand Up @@ -160,18 +162,18 @@ init_pg_db() {
# Returns:
# None
# Example:
# USER="user" SCHEMA="schema" DB="db" HOST="host" PORT="port" grant_permissions_for_schema
grant_permissions_for_schema() {
# USER="user" SCHEMA="schema" DB="db" HOST="host" PORT="port" grant_permissions_for_local_db_schema
grant_permissions_for_local_db_schema() {
local user=${USER-$DB_USER} schema=${SCHEMA-$DB_SCHEMA} db=${DB-$DB_NAME} host=${HOST-$DB_HOST} port=${PORT-$DB_PORT}
tlog "Granting permissions to user '${user}' on schema '$schema' in database '$db' on host '$host' and port '$port'..."
psql -h ${host} -p ${port} -U ${postgres_admin_user} -d ${db} -c "GRANT ALL PRIVILEGES ON SCHEMA ${schema} TO ${user};"
psql -h ${host} -p ${port} -U ${postgres_admin_user} -d ${db} -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${schema} TO ${user};"
psql -h ${host} -p ${port} -U ${postgres_admin_user} -d ${db} -c "ALTER DEFAULT PRIVILEGES IN SCHEMA ${schema} GRANT ALL PRIVILEGES ON TABLES TO ${user};"
psql -h ${host} -p ${port} -U ${postgres_admin_user} -d ${db} -c "GRANT CONNECT ON DATABASE ${db} TO ${user};"
psql -p ${port} -U ${POSTGRES_ADMIN_USER} -d ${db} -c "GRANT ALL PRIVILEGES ON SCHEMA ${schema} TO ${user};"
psql -p ${port} -U ${POSTGRES_ADMIN_USER} -d ${db} -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${schema} TO ${user};"
psql -p ${port} -U ${POSTGRES_ADMIN_USER} -d ${db} -c "ALTER DEFAULT PRIVILEGES IN SCHEMA ${schema} GRANT ALL PRIVILEGES ON TABLES TO ${user};"
psql -p ${port} -U ${POSTGRES_ADMIN_USER} -d ${db} -c "GRANT CONNECT ON DATABASE ${db} TO ${user};"
abhvsn marked this conversation as resolved.
Show resolved Hide resolved
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's add "-h /var/.sql" check the default unix socket path. let's be explicit that we are forcing unix socket use here


# Example usage of the functions
# waitForPostgresAvailability
# extract_postgres_db_params "postgresql://user:password@localhost:5432/dbname"
# init_pg_db
# USER="user" SCHEMA="schema" DB="db" HOST="host" PORT="port" grant_permissions_for_schema
# USER="user" SCHEMA="schema" DB="db" HOST="host" PORT="port" grant_permissions_for_local_db_schema
24 changes: 24 additions & 0 deletions deploy/docker/fs/opt/appsmith/postgres/appsmith_hba.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# This is a custom configuration for Embedded PostgreSQL for Appsmith.
# This file will be used to override the default pg_hba.conf file on restart.

# What is the meaning of this configuration?
# This configuration changes how PostgreSQL authenticates users
# connecting to the database. For user "postgres", we are allowing
# all connections from all addresses without any password on the unix
# socket. For all other users, we are allowing connections from IPv4
# and IPv6 with a password.

# TYPE DATABASE USER ADDRESS METHOD
# "local" is for Unix domain socket connections only
local all postgres trust
# IPv4 local connections:
host appsmith appsmith 127.0.0.1/32 scram-sha-256
host postgres appsmith 127.0.0.1/32 scram-sha-256
# IPv6 local connections:
host appsmith appsmith ::1/128 scram-sha-256
host postgres appsmith ::1/128 scram-sha-256
# Allow replication connections from localhost, by a user with the
# replication privilege.
local replication all scram-sha-256
host replication all 127.0.0.1/32 scram-sha-256
host replication all ::1/128 scram-sha-256
1 change: 1 addition & 0 deletions deploy/docker/tests/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
docker-compose.yml
186 changes: 186 additions & 0 deletions deploy/docker/tests/test-pg-auth.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
#!/bin/bash
set -o errexit
# set -x

source ./composes.sh

abhvsn marked this conversation as resolved.
Show resolved Hide resolved

# Function to update the APPSMITH_DB_URL in docker.env
# Once postgres is the default db, the APPSMITH_POSTGRES_DB_URL will be removed and this step won't be required anymore
# Check run-java.sh for more details why we need to update the APPSMITH_DB_URL to point to postgres
update_db_url() {
docker exec "${container_name}" bash -c "sed -i 's|^APPSMITH_DB_URL=mongodb|# &|' /appsmith-stacks/configuration/docker.env"
docker exec "${container_name}" bash -c "sed -i 's|^APPSMITH_POSTGRES_DB_URL=|APPSMITH_DB_URL=|' /appsmith-stacks/configuration/docker.env"
}

# Function to check if the Appsmith instance is up
is_appsmith_instance_ready() {
local max_retries=200
local retry_count=0
local response_code

while [ $retry_count -lt $max_retries ]; do
response_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost/health)
if [[ $response_code -eq 200 ]]; then
return 0
fi
echo "Waiting for Appsmith instance to be ready... (Attempt: $((retry_count + 1)))"
retry_count=$((retry_count + 1))
sleep 1
done
return 1
}

# Function to read the password from the PostgreSQL URL in docker.env.sh
get_appsmith_password() {
local password
password=$(docker exec "${container_name}" bash -c "grep -i 'APPSMITH_DB_URL' /appsmith-stacks/configuration/docker.env | sed -n 's/^.*\/\/appsmith:\([^@]*\)@.*$/\1/p'")
printf "%s" "$password"
}

# Function to check the read access to databases
check_user_datasource_access_with_auth() {
local password
local appsmith_user_local_access
local appsmith_user_remote_access
password=$(get_appsmith_password)
docker exec -i "${container_name}" bash -c "psql -h 127.0.0.1 -p 5432 -U appsmith -c '\l'" <<EOF
$password
EOF
appsmith_user_remote_access=$?
docker exec -i "${container_name}" bash -c "psql -p 5432 -U appsmith -c '\l'"
appsmith_user_local_access=$?
# Check if the Appsmith user has read access to databases
if [[ $appsmith_user_local_access -ne 0 && $appsmith_user_remote_access -eq 0 ]]; then
echo "appsmith user does not have read access to databases with local unix socket: ✅"
echo "appsmith user has read access to databases with local tcp socket: ✅"
local pg_user_local_access
local pg_user_remote_access
# Check if the postgres user has read access to databases with local unix socket
docker exec -i "${container_name}" bash -c "psql -p 5432 -U postgres -d appsmith -c '\l'"
pg_user_local_access=$?
# Check if the postgres user does not have read access to databases with local tcp socket
docker exec -i "${container_name}" bash -c "psql -h 127.0.0.1 -p 5432 -U postgres -d appsmith -c '\l'"
pg_user_remote_access=$?
if [[ $pg_user_local_access -eq 0 && $pg_user_remote_access -ne 0 ]]; then
echo "postgres user has read access to databases with local unix socket: ✅"
echo "postgres user does not have read access to databases with local tcp socket: ✅"
return 0
elif [[ $pg_user_local_access -ne 0 ]]; then
echo "postgres user does not have read access to databases with local unix socket: ❌"
elif [[ $pg_user_remote_access -eq 0 ]]; then
echo "postgres user has read access to databases with local tcp socket: ❌"
fi
elif [[ $appsmith_user_local_access -eq 0 ]]; then
echo "appsmith user has read access to databases with local unix socket: ❌"
elif [[ $appsmith_user_remote_access -ne 0 ]]; then
echo "appsmith user does not have read access to databases with local tcp socket: ❌"
fi
return 1
}
abhvsn marked this conversation as resolved.
Show resolved Hide resolved

# Function to check if the Appsmith user has read access to databases
check_user_datasource_access_wo_auth() {
docker exec "${container_name}" bash -c "psql -h 127.0.0.1 -p 5432 -U postgres -c '\l'"
if [[ $? -eq 0 ]]; then
return 0
fi
return 1
}

# Test to check if the postgres auth is enabled after upgrading from 1.47 to local image
# Expectation:
# 1. Appsmith instance should be able to upgrade from v1.47 to local image
# 6. Postgres user should have read access to databases with local unix socket
# 7. Postgres user should not have read access to databases with tcp socket
# 4. Appsmith user should not have read access to databases with local unix socket
# 5. Appsmith user should have read access to databases with tcp socket
test_postgres_auth_enabled_upgrade_from_147tolocal() {
# Steps:
# 1. Start the Appsmith 1.47 instance
# 2. Check if the Appsmith instance is up
# 3. Check if the postgres user has read access to databases
# 4. Update the APPSMITH_DB_URL in docker.env to point to postgres
# 5. Start the Appsmith local image
# 6. Check if the Appsmith instance is up
# 7. Check if the Appsmith user has read access to databases
# 8. Check if the postgres user has read access to databases
echo "############################################################"
echo "Starting ${FUNCNAME[0]}"

cleanup
abhvsn marked this conversation as resolved.
Show resolved Hide resolved
# appsmith v1.47 does not have postgres auth enabled
echo "Starting Appsmith 147"
compose_appsmith_version v1.47
# Wait until postgres to come up
until docker exec "${container_name}" pg_isready; do
echo "Waiting for postgres to be up..."
sleep 5
done

# Check if the Appsmith instance is up
if is_appsmith_instance_ready; then
echo "Appsmith instance is up and running."

# Check if the postgres user has read access to databases
if check_user_datasource_access_wo_auth; then
echo "postgres user has read access to databases: ✅"
else
echo "postgres user does not have read access to databases: ❌"
abhvsn marked this conversation as resolved.
Show resolved Hide resolved
exit 1
fi
else
echo "Appsmith instance failed to start."
exit 1
fi

# Update the APPSMITH_DB_URL in docker.env to point to postgres to initialise appsmith user and schema when the container with local image is started
update_db_url
echo "Remove container to reuse the same volume for local image"
docker compose down --timeout 30 # wait upto timeout for graceful shutdown.
# ensure the container exists before trying to remove it
docker compose ps -q "${container_name}" && \
docker compose rm -fsv "${container_name}" || \
echo "Container "${container_name}" does not exist."

echo "Starting Appsmith local to check the auth"
compose_appsmith_local

MAX_RETRIES=10
RETRYSECONDS=5
retry_count=0

while true; do
retry_count=$((retry_count + 1))
if docker exec "${container_name}" pg_isready &&
docker exec "${container_name}" bash -c 'cat /appsmith-stacks/data/postgres/main/PG_VERSION' = "14"; then
break
abhvsn marked this conversation as resolved.
Show resolved Hide resolved
fi
if [ $retry_count -le $MAX_RETRIES ]; then
echo "Waiting for postgres to be up..."
sleep $RETRYSECONDS
else
echo "Test ${FUNCNAME[0]} Failed"
exit 1
fi
done
abhvsn marked this conversation as resolved.
Show resolved Hide resolved

# Check if the Appsmith instance is up
if is_appsmith_instance_ready; then
echo "Appsmith instance is up and running."

# Check if the Appsmith user has read access to databases
if check_user_datasource_access_with_auth; then
echo "Test ${FUNCNAME[0]} Passed ✅"
exit 0
fi
else
echo "Appsmith instance failed to start."
fi

echo "Test ${FUNCNAME[0]} Failed ❌"
}

container_name="appsmith-docker-test"

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test for restart of local

test_postgres_auth_enabled_upgrade_from_147tolocal
abhvsn marked this conversation as resolved.
Show resolved Hide resolved
17 changes: 12 additions & 5 deletions scripts/local_testing.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ display_help()
echo "If --local or -l is passed, it will build with local changes"
echo "---------------------------------------------------------------------------------------"
echo
echo "Syntax: $0 [-h] [-l] [-r [remote_url]] [branch_name] [cs_url]"
echo "Syntax: $0 [-h] [-l] [-r [remote_url]] [branch_name] [tag] [cs_url]"
echo "options:"
echo "-h Print this help"
echo "-l or --local Use the local codebase and not git"
Expand Down Expand Up @@ -50,20 +50,23 @@ if [[ ($LOCAL == true) ]]
then
pretty_print "Setting up instance with local changes"
BRANCH=release
cs_url=$2
tag=$2
cs_url=$3
abhvsn marked this conversation as resolved.
Show resolved Hide resolved
elif [[ ($REMOTE == true) ]]
then
pretty_print "Setting up instance with remote repository branch ..."
REMOTE_REPOSITORY_URL=$2
REMOTE_BRANCH=$3
tag=$4
pretty_print "Please ignore if the following error occurs: remote remote_origin_for_local_test already exists."
git remote add remote_origin_for_local_test $REMOTE_REPOSITORY_URL || git remote set-url remote_origin_for_local_test $REMOTE_REPOSITORY_URL
git fetch remote_origin_for_local_test
git checkout $REMOTE_BRANCH
git pull remote_origin_for_local_test $REMOTE_BRANCH
else
BRANCH=$1
cs_url=$2
tag=$2
cs_url=$3
pretty_print "Setting up instance to run on branch: $BRANCH"
cd "$(dirname "$0")"/..
git fetch origin $BRANCH
Expand All @@ -72,6 +75,10 @@ else
pretty_print "Local branch is now up to date. Starting server build ..."
fi

if [[ -z "$tag" ]]; then
tag=latest
fi
pretty_print "Building Appsmith with tag: $tag"
edition=ce
if [[ "$(git remote get-url origin)" == *"/appsmith-ee"* ]]; then
edition=ee
Expand Down Expand Up @@ -106,12 +113,12 @@ pretty_print "RTS build successful. Starting Docker build ..."

popd
bash "$(dirname "$0")/generate_info_json.sh"
docker build -t appsmith/appsmith-ce:local-testing \
docker build -t appsmith/appsmith-local-$edition:$tag \
--build-arg BASE="appsmith/base-$edition:release" \
--build-arg APPSMITH_CLOUD_SERVICES_BASE_URL="${cs_url:-https://release-cs.appsmith.com}" \
. \
> /dev/null
pretty_print "Docker image build successful. Triggering run now ..."

(docker stop appsmith || true) && (docker rm appsmith || true)
docker run -d --name appsmith -p 80:80 -v "$PWD/stacks:/appsmith-stacks" appsmith/appsmith-ce:local-testing && sleep 15 && pretty_print "Local instance is up! Open Appsmith at http://localhost! "
docker run -d --name appsmith -p 80:80 -v "$PWD/stacks:/appsmith-stacks" appsmith/appsmith-local-$edition:$tag && sleep 15 && pretty_print "Local instance is up! Open Appsmith at http://localhost! "
abhvsn marked this conversation as resolved.
Show resolved Hide resolved
Loading