Merge branch 'main' into prs/add-github-releases

.envrc

@@ -0,0 +1 @@
+unset SSH_AUTH_SOCK

.gitignore

@@ -88,6 +88,9 @@ cython_debug/
 # Project specific
 data.pickle
 config.yaml
+osism_configuration.yaml
+osism_images.yaml
+osism_secrets.yaml
 .ceph
 .k8s
 .ssh

Makefile

@@ -34,7 +34,10 @@ help: ## Display this help message
 	@echo -e '\n${COLOUR_RED}Usage: make <command>${COLOUR_END}'
 	@cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]'  | \
 	    awk -F ':.*?## ' 'NF==2 {printf "  %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}'
-	@echo -e '${COLOUR_RED}OSISM helperscript usage: make <command>${COLOUR_END}'
+	@echo -e '${COLOUR_RED}Container related commands${COLOUR_END}'
+	@cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]'  | \
+	    awk -F ':.*?#container# ' 'NF==2 {printf "  %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}'
+	@echo -e '${COLOUR_RED}OSISM helperscripts${COLOUR_END}'
 	@cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]'  | \
 	    awk -F ':.*?#osism# ' 'NF==2 {printf "  %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}'
 
@@ -66,53 +69,55 @@ check-radoslib: ## Checks if radoslib is installed and if it contains the right
 	./scripts/check_local_rados_lib_installation.sh ${RADOSLIB_VERSION}
 
 .PHONY: build-local-rookify
-build-local-rookify: ## This builds rookify into .venv/bin/rookify
+build-local-rookify: setup ## This builds rookify into .venv/bin/rookify
 	source .venv/bin/activate && pip install -e .
 
-.PHONY: build-container
-build-container: ## Build container from Dockerfile, add e.g. ROOKIFY_VERSION=0.0.1 to specify the version. Default value is 0.0.0.dev1
-	${CONTAINERCMD} build --build-arg ROOKIFY_VERSION=$(ROOKIFY_VERSION) --target rookify -t rookify:latest -f Dockerfile .
-
 .PHONY: run-local-rookify
-run-local-rookify: ## Runs rookify in the local development environment (requires setup-venv)
-	source ./.venv/bin/activate && pip install -e . && rookify
-
-.PHONY: run-rookify
-run-rookify: ## Runs rookify in the container
-	docker exec -it rookify-dev /app/rookify/.venv/bin/rookify
-
-.PHONY: get-testbed-configs-for-rookify-testing
-get-testbed-configs-for-rookify-testing: ## Gets the needed config (like .kube, /etc/ceph and so on) from the testbed
-	bash ./scripts/get_configs_from_testbed.sh
+run-local-rookify: build-local-rookify ## Runs rookify in the local development environment (requires setup-venv)
+	./.venv/bin/rookify
 
 .PHONY: run-tests-locally
-run-tests-locally: ## Runs the tests in the tests directory. NB: check that your local setup is connected through vpn to the testbed!
+run-tests-locally: setup-venv ## Runs the tests in the tests directory. NB: check that your local setup is connected through vpn to the testbed!
 	.venv/bin/python -m pytest
 
+##
+# Add container related commands here (so they appear below the container header)
+# Note: use #container# so command appear under header in menu
+##
+
 .PHONY: run-tests
-run-tests: up ## Runs the tests in the container
+run-tests: up #container# Runs the tests in the container
 	${CONTAINERCMD} exec -it rookify-dev bash -c "/app/rookify/.venv/bin/python -m pytest"
 
+.PHONY: run-rookify
+run-rookify: up #container# Runs rookify in the container
+	${CONTAINERCMD} exec -it rookify-dev /app/rookify/.venv/bin/rookify
+
 .PHONY: enter
-enter: ## Enter the container
+enter: up #container# Enter the container
 	${CONTAINERCMD} exec -it rookify-dev bash
 
 .PHONY: logs
-logs: ## Logs the container
+logs: #container# Logs the container
 	${CONTAINERCMD} logs -f rookify-dev
 
 .PHONY: down
-down: ## Remove the containers as setup by docker-compose.yml
+down: #container# Remove the containers as setup by docker-compose.yml
 	${CONTAINERCMD} compose down
 
 .PHONY: up
-up: ## Sets up the container as specified in docker-compose.yml and opens a bash terminal
+up: #container# Sets up the container as specified in docker-compose.yml and opens a bash terminal
 	${CONTAINERCMD} compose up -d
 
+.PHONY: build-container
+build-container: #container# Build container from Dockerfile only, add e.g. ROOKIFY_VERSION=0.0.1 to specify the version. Default value is 0.0.0.dev1
+	${CONTAINERCMD} build --build-arg ROOKIFY_VERSION=$(ROOKIFY_VERSION) --target rookify -t rookify:latest -f Dockerfile .
+
 ##
-# Add osism specific scripts below here (so they appear below helper header)
+# Add osism specific scripts below here
+# Note: use #osism# so command appear under header in menu
 ##
 
 .PHONY: get-config
 get-config: #osism# Gets configuration files from the OSISM testbed
-	./scripts/osism/get_osism_configs_from_testbed.sh
+	./scripts/osism/get_configs_from_testbed.sh

README.md

@@ -14,11 +14,12 @@ Rookify is designed to facilitate a smooth and efficient transition for existing
 - A functioning Ceph cluster deployed via traditional methods.
   - __TODO:__ List supported methods
 - Access to a Kubernetes cluster with sufficient resources to host the migrated Ceph cluster.
-  - Kubernetes nodes should be rolled out at least on the OSD nodes
+    - Kubernetes nodes should be rolled out at least on the monitor nodes
 - Rook operator version 1.13 or higher installed in the Kubernetes cluster.
 - _local development enivornment_ requires radoslib version 2.0.0 installed
 
 ## Installation
+
 1. Clone the repository:
 ```bash
 git clone https://github.com/SovereignCloudStack/rookify
@@ -29,24 +30,92 @@ git clone https://github.com/SovereignCloudStack/rookify
 cd rookify
 ```
 
-3. To install the local development environment
+3. Check if your host has the correct "radoslib" library installed (if not: then install radoslib version 2.0.0):
+```bash
+make check-radoslib
+```
+
+4. To install the local development environment
 (_Note: This will install pre-commit in your local user context_):
 ```bash
 make setup
 ```
 
-4. To install the container-based environment
+5. To install the container-based environment
 ```bash
 make build-container
-docker run -ti --mount type=bind,source="$(pwd)",target=/app/rookify/src/,readonly --workdir=/app/rookify/src rookify:latest
 ```
 
 ## Usage
-__TODO__
 
-Type `make` to get a list of available development specific commands.
+**NOTE**: for testing purposes the [OSISM Testbed](https://github.com/osism/testbed) is used. The `Makefile` and example configuration (`config.example.osism.yaml`) have built in helper functions and examples for this test setup.
+
+### Copy and adapt configuration files
+
+Choose one of the configuration-examples found in the root of the working directory and copy it to `config.yml`:
+
+```
+ls config.example.*
+# there is a specific example config for the OSISM testbed: config.example.osism.yaml
+cp config.example.yml config.yaml
+```
+
+_Adapt the config.yml to your need_: especially enter the correct paths for ssh-keys, kubernetes configuration and ceph configuration (all these configuration files need to be provided!).
+
+### Provide needed configuration files from target servers
+
+Copy the needed configuration-files from the servers that need to be migrated from ceph to rook.
+
+_Provide needed configuration files as written in the config.yml._ At least required are:
+- ./ceph/ceph.conf (typically found in `/etc/ceph/` on a testbednode)
+- ./ceph/ceph.admin.keyring (typically found in `/etc/ceph/` on a testbednode)
+- kubernetes config of user (e.g. found in `~/.kube/config`)
+- ssh key of the server (found at `./terraform/.id.rsa` in the `testbed` directory)
+
+Note:
+- for the testbed there is a helper script to download the configs from the testbed. These helperscripts need correct `.ssh/config` entries to work (take a look at [scripts/osism/get_configs_from_testbed](osism/get_configs_from_testbed.sh) for an example).
+- the helper scripts are merely there to help for testing with the [OSISM testbed](https://github.com/osism/testbed) and might not suit your purposes.
+
+### Run Rookify
+
+Now decide on how to run rookify. Either run it from within a container or locally:
+
+```
+run-local-rookify
+# or
+run-rookify
+```
+
+Then sit back and wait. Check the [troubleshooting](#troubleshooting) and [support](#support) sections if any issues occur.
+
+For other options you can also check the makefile: Type `make` to get a list of available commands.
+
+### Check status of migration
+
+Use `rookify --show-states` argument to let each module check for its state in the migration
+
+```bash
+.venv/bin/rookify --show-states
+ceph mon dump: Not analyzed yet
+ceph osd dump: Not analyzed yet
+OSD devices: Not analyzed yet
+```
+
+#### Troubleshooting
+
+**missing Ceph systemd targets:**
+- please consult your distribution's documentation in case documented Ceph systemd unit services or targets are missing
+- `cephadm install ceph-common ceph-base ceph-mon ceph-mgr ceph-mds radosgw` may help if supported
+
+**frozen state:**
+- if the rookify process freezes, check your connections. In the OSISM testbed especially check the vpn-connection (in testbed repository try `make vpn-*`)
+
+**ssh-issues:**
+- make sure the id-rsa keys are "clean" and do not contain unexpected strings like "\<\<EOF". E.g. call `ssh-keygen -p -N "" -f ssh.key` to convert and reformat the keyfile to the expected format.
+- allow direnv (`direnv allow`) to use `.envrc` or copy and execute the command from the file: this switches off the ssh-agent, which sometimes has too many keys loaded
 
 ## Support
+
 For issues, questions, or contributions, please open an issue or pull request in the GitHub repository. We welcome community feedback and contributions to enhance rookify.
 
 ## License

config.example.osism.yaml

@@ -42,10 +42,4 @@ rook:
 
 migration_modules:
 - example
-- create_rook_cluster
-- migrate_osds
-- migrate_osd_pools
-- migrate_mds
-- migrate_mds_pools
-- migrate_rgws
-- migrate_rgw_pools
+- osism_configuration

config.example.yaml

@@ -10,6 +10,12 @@ logging:
 ceph:
   config: ./.ceph/ceph.conf
   keyring: ./.ceph/ceph.client.admin.keyring
+  systemd_file_name_templates:
+    mds: "ceph-{fsid}@mds.{host}.service"
+    mgr: "ceph-{fsid}@mgr.{host}.service"
+    mon: "ceph-{fsid}@mon.{host}.service"
+    osd: "ceph-{fsid}@osd.{osd_id}.service"
+    rgw: "ceph-{fsid}@rgw.{host}.service"
 
 # fill in correct path to private key
 ssh:

scripts/osism/get_configs_from_testbed.sh

@@ -0,0 +1,28 @@
+# Helperscript: Gets configs from testbed
+#
+# NOTE: this is for test purposes and specific for the OISISM testbed
+# it requires .ssh/config to be set accordingly, e.g.:
+#
+# Host testbed-*
+#  StrictHostKeyChecking no
+#  IdentityFile <id_rsa of testbed>
+#  IdentitiesOnly yes
+#  user dragon
+#
+# Host testbed-manager
+#  Hostname <ip of manager of testbed>
+#
+# Host testbed-node-0
+#  Hostname 192.168.16.10
+#
+# Host testbed-node-1
+#   Hostname 192.168.16.11
+# Host testbed-node-2
+#  Hostname 192.168.16.12
+#
+
+# copy .kube to ./.k8s
+scp -r testbed-manager:.kube ./.k8s
+
+# copy /etc/ceph/ from node1 to ./.k8s
+ssh testbed-manager "docker cp cephclient:/etc/ceph /tmp/cephclientconfig" && scp -r testbed-manager:/tmp/cephclientconfig ./.ceph && ssh testbed-manager "rm -rf /tmp/cephclientconfig"

src/rookify/config.schema.yaml

@@ -10,6 +10,7 @@ logging:
 ceph:
   config: str()
   keyring: str()
+  systemd_file_name_templates: include("systemd_file_name_templates", required=False)
 
 ssh:
   private_key: str()
@@ -39,3 +40,10 @@ migration_modules: list(str())
 ssh_host:
     address: ip()
     user: str()
+---
+systemd_file_name_templates:
+    mds: str(required=False)
+    mgr: str(required=False)
+    mon: str(required=False)
+    osd: str(required=False)
+    rgw: str(required=False)

src/rookify/modules/ceph.py

@@ -16,6 +16,14 @@ def __init__(self, config: Dict[str, Any]):
         except rados.ObjectNotFound as err:
             raise ModuleException(f"Could not connect to ceph: {err}")
 
+        status_data = self.mon_command("status")
+
+        self._fsid = status_data["fsid"]
+
+        self._systemd_file_name_templates = config.get(
+            "systemd_file_name_templates", {}
+        )
+
     def __getattr__(self, name: str) -> Any:
         return getattr(self.__ceph, name)
 
@@ -32,7 +40,43 @@ def _json_command(self, handler: Any, *args: Any) -> Any:
 
         return data
 
-    def get_osd_pool_configurations_from_osd_dump(
+    def get_systemd_mds_file_name(self, host: str) -> str:
+        return self._get_systemd_template_file_name(
+            self._systemd_file_name_templates.get("mds", "ceph-mds.target"),
+            host,
+        )
+
+    def get_systemd_mgr_file_name(self, host: str) -> str:
+        return self._get_systemd_template_file_name(
+            self._systemd_file_name_templates.get("mgr", "ceph-mgr.target"),
+            host,
+        )
+
+    def get_systemd_mon_file_name(self, host: str) -> str:
+        return self._get_systemd_template_file_name(
+            self._systemd_file_name_templates.get("mon", "ceph-mon.target"),
+            host,
+        )
+
+    def get_systemd_osd_file_name(self, host: str, osd_id: int) -> str:
+        file_name_template: str = self._systemd_file_name_templates.get(
+            "osd", "ceph-osd@{0:d}.service".format(osd_id)
+        )
+
+        return file_name_template.format(fsid=self._fsid, host=host, osd_id=osd_id)
+
+    def get_systemd_rgw_file_name(self, host: str) -> str:
+        return self._get_systemd_template_file_name(
+            self._systemd_file_name_templates.get("mon", "ceph-radosgw.target"),
+            host,
+        )
+
+    def _get_systemd_template_file_name(
+        self, file_name_template: str, host: str
+    ) -> str:
+        return file_name_template.format(fsid=self._fsid, host=host)
+
+    def get_osd_pool_configurations_from_map(
         self, dump_data: Dict[str, Any]
     ) -> Dict[str, Any]:
         osd_pools = {osd_pool["pool_name"]: osd_pool for osd_pool in dump_data["pools"]}

src/rookify/modules/create_rook_resources/main.py

@@ -36,7 +36,7 @@ def __create_configmap_definition(self) -> None:
         configmap_data = {
             "data": configmap_mon_list,
             "mapping": json.dumps({"node": mapping}),
-            "maxMonId": "{0:d}".format(len(state_data["mon"]["dump"]["mons"])),
+            "maxMonId": "{0:d}".format(len(state_data["report"]["monmap"]["mons"])),
         }
 
         configmap = kubernetes.client.V1ConfigMap(

src/rookify/modules/migrate_mds/main.py

@@ -83,7 +83,10 @@ def execute(self) -> None:
 
     def _disable_mds(self, mds_host: str) -> None:
         result = self.ssh.command(
-            mds_host, "sudo systemctl disable --now ceph-mds.target"
+            mds_host,
+            "sudo systemctl disable --now {0}".format(
+                self.ceph.get_systemd_mds_file_name(mds_host)
+            ),
         )
 
         if result.failed:

src/rookify/modules/migrate_mds_pools/main.py

@@ -16,8 +16,8 @@ def preflight(self) -> None:
             "MigrateMdsPoolsHandler", "pools", default_value={}
         )
 
-        osd_pools = self.ceph.get_osd_pool_configurations_from_osd_dump(
-            state_data["osd"]["dump"]
+        osd_pools = self.ceph.get_osd_pool_configurations_from_map(
+            state_data["report"]["osdmap"]
         )
 
         for mds_fs_data in state_data["fs"]["ls"]:

src/rookify/modules/migrate_mgrs/main.py

@@ -44,7 +44,10 @@ def _migrate_mgr(self, mgr_host: str) -> None:
         self.logger.info("Migrating ceph-mgr daemon at host'{0}'".format(mgr_host))
 
         result = self.ssh.command(
-            mgr_host, "sudo systemctl disable --now ceph-mgr.target"
+            mgr_host,
+            "sudo systemctl disable --now {0}".format(
+                self.ceph.get_systemd_mgr_file_name(mgr_host)
+            ),
         )
 
         if result.failed: