# These initial groups are the only groups required to be modified. The
# additional groups are for more control of the environment.
[control]
# These hostname must be resolvable from your deployment host
192.168.1.40
192.168.1.38
192.168.1.36

# The above can also be specified as follows:
#control[01:03]     ansible_user=kolla

# The network nodes are where your l3-agent and loadbalancers will run
# This can be the same as a host in the control group
[network]
192.168.1.40
192.168.1.38
192.168.1.36

[compute]
192.168.1.28

[monitoring]


# When compute nodes and control nodes use different interfaces,
# you need to comment out "api_interface" and other interfaces from the globals.yml
# and specify like below:
#compute01 neutron_external_interface=eth0 api_interface=em1 storage_interface=em1 tunnel_interface=em1

[storage]
192.168.1.40
192.168.1.38
192.168.1.36
192.168.1.28

[deployment]
localhost       ansible_connection=local

kolla_base_distro: "centos"
kolla_install_type: "source"
network_interface: "eno1"
kolla_external_vip_interface: "br0"
neutron_external_interface: "veno1"
keepalived_virtual_router_id: "63"
kolla_internal_vip_address: "192.168.1.63"
kolla_external_vip_address: "10.245.121.63"

kolla_enable_tls_internal: "yes"
kolla_enable_tls_external: "yes"
kolla_enable_tls_backend: "yes"
rabbitmq_enable_tls: "yes"
kolla_copy_ca_into_containers: "yes"
openstack_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if kolla_enable_tls_external == 'yes' else '' }}"


glance_backend_ceph: "yes"
glance_backend_file: "no"
glance_enable_rolling_upgrade: "yes"

enable_cinder: "yes"

ceph_nova_user: "cinder"
cinder_backend_ceph: "yes"
cinder_backup_driver: "ceph"

nova_backend_ceph: "yes"

# Swift (radosgw) options:
enable_ceph_rgw: true # Feature from Xena onwards
enable_swift: "no" # Feature for swift on disk, not through ceph.
enable_swift_s3api: "yes"
enable_ceph_rgw_keystone: true
ceph_rgw_swift_compatibility: true
ceph_rgw_swift_account_in_url: true
enable_ceph_rgw_loadbalancer: true
ceph_rgw_hosts:
  - host: r1-710-40
    ip: 192.168.1.40
    port: 7480
  - host: r1-710-38
    ip: 192.168.1.38
    port: 7480
  - host: r1-710-36
    ip: 192.168.1.36
    port: 7480

docker_registry: "10.245.0.14"
docker_registry_insecure: "yes"
docker_registry_username: "kolla"

enable_mariabackup: "no"

enable_haproxy: "yes"

kolla_external_fqdn: "optional.fqdn.com"

echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$(lsb_release -rs)/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
curl -fsSL https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_"$(lsb_release -rs)"/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/devel_kubic_libcontainers_stable.gpg > /dev/null

# Update to fetch the package index for ceph added above
sudo apt-get update
sudo apt-get -qqy install podman
podman --version

MONITOR_IP=$1
CEPH_RELEASE=pacific

# Update to fetch the latest package index
sudo apt-get update

# Fetch most recent version of cephadm
curl --silent --remote-name --location https://github.com/ceph/ceph/raw/"$CEPH_RELEASE"/src/cephadm/cephadm
chmod +x cephadm

sudo ./cephadm add-repo --release "$CEPH_RELEASE"

# Update to fetch the package index for ceph added above
sudo apt-get update

# Install ceph-common and cephadm packages
sudo ./cephadm install ceph-common
sudo ./cephadm install

sudo mkdir -p /etc/ceph
sudo ./cephadm bootstrap --mon-ip "$MONITOR_IP"

# Turn on telemetry and accept Community Data License Agreement - Sharing
sudo ceph telemetry on --license sharing-1-0

sudo ceph -v
sudo ceph status

sudo ceph orch host ls
sudo ceph orch device ls --refresh
sudo ceph orch apply osd --all-available-devices

# Create pool for Cinder
sudo ceph osd pool create volumes
sudo rbd pool init volumes

# Create pool for Cinder Backup
sudo ceph osd pool create backups
sudo rbd pool init backups

# Create pool for Glance
sudo ceph osd pool create images
sudo rbd pool init images

# Create pool for Nova
sudo ceph osd pool create vms
sudo rbd pool init vms

# Create pool for Gnocchi
#sudo ceph osd pool create metrics
#sudo rbd pool init metrics

# Enable swift, 1 for refstack, 3 for production
source "$HOME"/swift_settings.sh 3 

# Get cinder and cinder-backup ready
sudo mkdir -p /etc/kolla/config/cinder/cinder-backup
sudo chown -R ubuntu:ubuntu /etc/kolla/config/
sudo cp /etc/ceph/ceph.conf /etc/kolla/config/cinder/cinder-backup/ceph.conf
sudo ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups' mgr 'profile rbd pool=backups' > /etc/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring
sudo ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd pool=images' > /etc/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring
sudo sed -i $'s/\t//g' /etc/kolla/config/cinder/cinder-backup/ceph.conf
sudo sed -i $'s/\t//g' /etc/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring
sudo sed -i $'s/\t//g' /etc/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring

# Get cinder-volume ready
sudo mkdir -p /etc/kolla/config/cinder/cinder-volume
sudo chown -R ubuntu:ubuntu /etc/kolla/config/
sudo cp /etc/ceph/ceph.conf /etc/kolla/config/cinder/cinder-volume/ceph.conf
sudo ceph auth get-or-create client.cinder > /etc/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring
sudo sed -i $'s/\t//g' /etc/kolla/config/cinder/cinder-volume/ceph.conf
sudo sed -i $'s/\t//g' /etc/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring

# Get glance ready
sudo mkdir -p /etc/kolla/config/glance
sudo chown -R ubuntu:ubuntu /etc/kolla/config/
sudo cp /etc/ceph/ceph.conf /etc/kolla/config/glance/ceph.conf
sudo ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=images' > /etc/kolla/config/glance/ceph.client.glance.keyring
sudo sed -i $'s/\t//g' /etc/kolla/config/glance/ceph.conf
sudo sed -i $'s/\t//g' /etc/kolla/config/glance/ceph.client.glance.keyring

# Get nova ready
sudo mkdir -p /etc/kolla/config/nova
sudo chown -R ubuntu:ubuntu /etc/kolla/config/
sudo cp /etc/ceph/ceph.conf /etc/kolla/config/nova/ceph.conf
sudo ceph auth get-or-create client.cinder > /etc/kolla/config/nova/ceph.client.cinder.keyring
sudo sed -i $'s/\t//g' /etc/kolla/config/nova/ceph.conf
sudo sed -i $'s/\t//g' /etc/kolla/config/nova/ceph.client.cinder.keyring

# Get Gnocchi ready
#sudo mkdir -p  /etc/kolla/config/gnocchi
#sudo chown -R ubuntu:ubuntu /etc/kolla/config/
#sudo cp /etc/ceph/ceph.conf /etc/kolla/config/gnocchi/ceph.conf
#sudo ceph auth get-or-create client.gnocchi mon 'profile rbd' osd 'profile rbd pool=metrics' mgr 'profile rbd pool=metrics' > /etc/kolla/config/gnocchi/ceph.client.gnocchi.keyring
#sudo sed -i $'s/\t//g' /etc/kolla/config/gnocchi/ceph.conf
#sudo sed -i $'s/\t//g' /etc/kolla/config/gnocchi/ceph.client.gnocchi.keyring

# Verify all permissions are correct.
sudo chown -R ubuntu:ubuntu /etc/kolla/config/
sudo ceph status

#!/bin/bash

set -euxo pipefail

NUM_OF_WHOS=$1
sudo ceph orch apply rgw osiasswift --port=7480 --placement="$NUM_OF_WHOS" # Default port results in port conflict and fails.
sudo ceph dashboard set-rgw-api-ssl-verify False
sudo ceph orch apply mgr "$HOSTNAME"

if [[ $(grep -c ceph_rgw_keystone_password /etc/kolla/passwords.yml) -eq 1 ]]
then
    ceph_rgw_pass=$( grep ceph_rgw_keystone_password /etc/kolla/passwords.yml | cut -d':' -f2 | xargs )
    rgw_keystone_admin_user="ceph_rgw"
else
    ceph_rgw_pass=$( grep keystone_admin_password /etc/kolla/passwords.yml | cut -d':' -f2 | xargs )
    rgw_keystone_admin_user="admin"
fi

internal_url=$( grep ^kolla_internal_vip_address: /etc/kolla/globals.yml | cut -d':' -f2 | xargs )

# https://docs.ceph.com/en/latest/radosgw/keystone/#integrating-with-openstack-keystone
# https://www.spinics.net/lists/ceph-users/msg64137.html
# The "WHO" field in the "ceph config set" needs to be "client.rgw.default" NOT
# "client.radosgw.gateway". This can be verified by issuing "ceph config dump"
# Additionally, the name of all of the gateways need to be present.

WHO_IS=""
NUM_WHO_IS=$(echo "$WHO_IS" | wc -w)
while [[ "$NUM_WHO_IS" -lt "$NUM_OF_WHOS" ]]
do
    WHO_IS="$(sudo ceph auth ls | grep client.rgw | grep client)" || true
    echo "Waiting..."
    sleep 10
    NUM_WHO_IS=$(echo "$WHO_IS" | wc -w)
done

WHO_IS="client.rgw.default $WHO_IS"
echo "RGW CLIENTS: $WHO_IS"
for WHO in $WHO_IS; do
    sudo ceph config set "$WHO" rgw_keystone_api_version 3
    sudo ceph config set "$WHO" rgw_keystone_url https://"$internal_url":35357
    sudo ceph config set "$WHO" rgw_keystone_accepted_admin_roles "admin, ResellerAdmin"
    sudo ceph config set "$WHO" rgw_keystone_accepted_roles "_member_, member, admin, ResellerAdmin"
    sudo ceph config set "$WHO" rgw_keystone_implicit_tenants true # Implicitly create new users in their own tenant with the same name when authenticating via Keystone. Can be limited to s3 or swift only.
    sudo ceph config set "$WHO" rgw_keystone_admin_user "$rgw_keystone_admin_user"
    sudo ceph config set "$WHO" rgw_keystone_admin_password "$ceph_rgw_pass" # Got from the passwords.yml
    sudo ceph config set "$WHO" rgw_keystone_admin_project service
    sudo ceph config set "$WHO" rgw_keystone_admin_domain default
    sudo ceph config set "$WHO" rgw_keystone_verify_ssl false
    sudo ceph config set "$WHO" rgw_content_length_compat true
    sudo ceph config set "$WHO" rgw_enable_apis "s3, swift, swift_auth, admin"
    sudo ceph config set "$WHO" rgw_s3_auth_use_keystone true
    sudo ceph config set "$WHO" rgw_enforce_swift_acls true
    sudo ceph config set "$WHO" rgw_swift_account_in_url true
    sudo ceph config set "$WHO" rgw_swift_versioning_enabled true
    sudo ceph config set "$WHO" rgw_verify_ssl true
done


# Redeploy your rgw daemon
sudo ceph orch restart rgw.osiasswift
HOSTNAMES=$(sudo ceph orch host ls | grep -v HOST | awk '{print $1}' | tr '\n' ',')
sudo ceph orch apply mgr "$HOSTNAMES" # Add back-up mgr hosts