feat: add options to configure utility only from cli

nightingale/cli.py

@@ -1,15 +1,16 @@
 import logging
+import tomllib
 
 import click
 import click_pathlib
+from pydantic import TypeAdapter
 
 from .config import Config
 from .loader import DataLoader
 from .mapper import OCDSDataMapper
 from .publisher import DataPublisher
 from .writer import DataWriter
 
-# Set up logging
 logger = logging.getLogger(__name__)
 
 
@@ -24,6 +25,20 @@ def setup_logging(loglevel):
     )
 
 
+def load_config(config_file):
+    """
+    Load configuration data from a TOML file.
+
+    :param config_file: Path to the configuration file.
+    :return: Dictionary with configuration data.
+    """
+    try:
+        with open(config_file, "rb") as f:
+            return tomllib.load(f)
+    except tomllib.TOMLDecodeError:
+        raise click.ClickException(f"Error decoding TOML from {config_file}.")
+
+
 @click.command()
 @click.option(
     "--config",
@@ -40,24 +55,91 @@ def setup_logging(loglevel):
     default="INFO",
     help="Set the logging level",
 )
-def run(config_file, package, validate_mapping, loglevel):
+@click.option("--datasource", type=str, help="Datasource connection string")
+@click.option("--mapping-file", type=click_pathlib.Path(exists=True), help="Mapping file path")
+@click.option("--ocid-prefix", type=str, help="OCID prefix")
+@click.option("--selector", type=str, help="Selector")
+@click.option("--force-publish", is_flag=True, help="Force publish")
+@click.option("--publisher", type=str, help="Publisher")
+@click.option("--base-uri", type=str, help="Base URI")
+@click.option("--version", type=str, help="Version")
+@click.option("--publisher-uid", type=str, help="Publisher UID")
+@click.option("--publisher-scheme", type=str, help="Publisher scheme")
+@click.option("--publisher-uri", type=str, help="Publisher URI")
+@click.option("--extensions", type=str, multiple=True, help="Extensions")
+@click.option("--output-directory", type=click_pathlib.Path(exists=True), help="Output directory")
+def run(
+    config_file,
+    package,
+    validate_mapping,
+    loglevel,
+    datasource,
+    mapping_file,
+    ocid_prefix,
+    selector,
+    force_publish,
+    publisher,
+    base_uri,
+    version,
+    publisher_uid,
+    publisher_scheme,
+    publisher_uri,
+    extensions,
+    output_directory,
+):
+    """
+    Run the data transformation process.
+
+    :param config_file: Path to the configuration file.
+    :param package: Flag to indicate whether to package the data.
+    :param validate_mapping: Flag to indicate whether to validate the mapping template.
+    :param loglevel: Logging level.
+    """
     setup_logging(loglevel)
-    logger.info("Start transforming")
+    logger.info("Starting data transformation")
 
     try:
-        config = Config.from_file(config_file)
+        logger.debug(f"Loading configuration from {config_file}")
+        config_data = load_config(config_file)
+
+        # Apply CLI overrides
+        if datasource:
+            config_data["datasource"] = {"connection": datasource}
+        if mapping_file or ocid_prefix or selector or force_publish:
+            config_data["mapping"] = {
+                "file": mapping_file or config_data["mapping"]["file"],
+                "ocid_prefix": ocid_prefix or config_data["mapping"]["ocid_prefix"],
+                "selector": selector or config_data["mapping"]["selector"],
+                "force_publish": force_publish
+                if force_publish is not None
+                else config_data["mapping"].get("force_publish", False),
+            }
+        if publisher or base_uri or version or publisher_uid or publisher_scheme or publisher_uri or extensions:
+            config_data["publishing"] = {
+                "publisher": publisher or config_data["publishing"]["publisher"],
+                "base_uri": base_uri or config_data["publishing"]["base_uri"],
+                "version": version or config_data["publishing"].get("version", ""),
+                "publisher_uid": publisher_uid or config_data["publishing"].get("publisher_uid", ""),
+                "publisher_scheme": publisher_scheme or config_data["publishing"].get("publisher_scheme", ""),
+                "publisher_uri": publisher_uri or config_data["publishing"].get("publisher_uri", ""),
+                "extensions": list(extensions) if extensions else config_data["publishing"].get("extensions", []),
+            }
+        if output_directory:
+            config_data["output"] = {"directory": output_directory}
+
+        # Validate final configuration
+        config = TypeAdapter(Config).validate_python(config_data)
         mapper = OCDSDataMapper(config)
         writer = DataWriter(config.output)
-        logger.info("MappingTemplate data...")
+
         ocds_data = mapper.map(DataLoader(config.datasource), validate_mapping=validate_mapping)
 
         if package:
             logger.info("Packaging data...")
             packer = DataPublisher(config.publishing)
             ocds_data = packer.package(ocds_data)
-
         logger.info("Writing data...")
-        writer.write({"releases": ocds_data} if not package else ocds_data)
+        writer.write(ocds_data)
         logger.info("Data transformation completed successfully")
 
     except Exception as e:

nightingale/mapper.py

@@ -50,8 +50,11 @@ def map(self, loader: Any, validate_mapping: bool = False) -> list[dict[str, Any
         """
         config = self.config.mapping
         mapping = MappingTemplate(config)
+        logger.info("MappingTemplate data loaded")
         data = loader.load(config.selector)
+        logger.info("Source data is loaded...")
         if validate_mapping:
+            logger.info("Validating mapping template...")
             validator = MappingTemplateValidator(loader, mapping)
             validator.validate_data_elements()
             validator.validate_selector(data[0])

nightingale/writer.py

@@ -6,7 +6,7 @@
 from .utils import produce_package_name
 
 
-def new_name(package: dict) -> str:
+def new_name(package: dict | list) -> str:
     """
     Generate a new name for the package based on its published date.
 
@@ -15,7 +15,11 @@ def new_name(package: dict) -> str:
     :return: The generated package name.
     :rtype: str
     """
-    date = package.get("publishedDate", datetime.now().isoformat())
+
+    if isinstance(package, list):
+        date = datetime.now().isoformat()
+    else:
+        date = package.get("publishedDate", datetime.now().isoformat())
     return produce_package_name(date)
 
 
@@ -59,7 +63,7 @@ def get_output_path(self, package: dict) -> Path:
         base = self.make_dirs()
         return base / new_name(package)
 
-    def write(self, package: dict) -> None:
+    def write(self, package: list | dict) -> None:
         """
         Write the release package to disk.
 

tests/test_cli.py

@@ -14,6 +14,7 @@ def setUp(self):
         self.runner = CliRunner()
         self.temp_dir = tempfile.TemporaryDirectory()
         self.config_path = Path(self.temp_dir.name) / "test_config.toml"
+        self.invalid_config_path = Path(self.temp_dir.name) / "invalid_test_config.toml"
         self.config_data = """
         [datasource]
         connection = "test_connection"
@@ -35,15 +36,24 @@ def setUp(self):
         with open(self.config_path, "w") as f:
             f.write(self.config_data)
 
+        self.invalid_config_data = """
+        [datasource
+        connection = "test_connection"
+        """
+        with open(self.invalid_config_path, "w") as f:
+            f.write(self.invalid_config_data)
+
     def tearDown(self):
         self.temp_dir.cleanup()
 
+    @patch("nightingale.cli.Config.from_file")
     @patch("nightingale.cli.OCDSDataMapper")
     @patch("nightingale.cli.DataLoader")
     @patch("nightingale.cli.DataWriter")
     @patch("nightingale.cli.DataPublisher")
-    def test_run_without_package(self, mock_publisher, mock_writer, mock_loader, mock_mapper):
+    def test_run_without_package(self, mock_publisher, mock_writer, mock_loader, mock_mapper, mock_config):
         # Setup mocks
+        mock_config.return_value = MagicMock()
         mock_mapper_instance = MagicMock()
         mock_mapper.return_value = mock_mapper_instance
 
@@ -53,23 +63,25 @@ def test_run_without_package(self, mock_publisher, mock_writer, mock_loader, moc
         mock_writer_instance = MagicMock()
         mock_writer.return_value = mock_writer_instance
 
-        mock_mapper_instance.map.return_value = {"dummy_data": "data"}
+        mock_mapper_instance.map.return_value = [{"dummy_data": "data"}]
 
         result = self.runner.invoke(run, ["--config", str(self.config_path), "--loglevel", "INFO"])
 
         self.assertEqual(result.exit_code, 0)
         mock_mapper.assert_called_once()
         mock_loader.assert_called_once()
         mock_writer.assert_called_once()
-        mock_writer_instance.write.assert_called_once_with({"releases": {"dummy_data": "data"}})
+        mock_writer_instance.write.assert_called_once_with([{"dummy_data": "data"}])
         mock_publisher.assert_not_called()
 
+    @patch("nightingale.cli.Config.from_file")
     @patch("nightingale.cli.OCDSDataMapper")
     @patch("nightingale.cli.DataLoader")
     @patch("nightingale.cli.DataWriter")
     @patch("nightingale.cli.DataPublisher")
-    def test_run_with_package(self, mock_publisher, mock_writer, mock_loader, mock_mapper):
+    def test_run_with_package(self, mock_publisher, mock_writer, mock_loader, mock_mapper, mock_config):
         # Setup mocks
+        mock_config.return_value = MagicMock()
         mock_mapper_instance = MagicMock()
         mock_mapper.return_value = mock_mapper_instance
 
@@ -101,11 +113,13 @@ def test_setup_logging(self):
 
         self.assertIn("This is a debug message", log.output[0])
 
+    @patch("nightingale.cli.Config.from_file")
     @patch("nightingale.cli.OCDSDataMapper")
     @patch("nightingale.cli.DataLoader")
     @patch("nightingale.cli.DataWriter")
-    def test_run_mapping_crash(self, mock_writer, mock_loader, mock_mapper):
+    def test_run_mapping_crash(self, mock_writer, mock_loader, mock_mapper, mock_config):
         # Setup mocks
+        mock_config.return_value = MagicMock()
         mock_mapper_instance = MagicMock()
         mock_mapper.return_value = mock_mapper_instance
 
@@ -124,6 +138,11 @@ def test_run_mapping_crash(self, mock_writer, mock_loader, mock_mapper):
         mock_mapper.assert_called_once()
         mock_loader.assert_called_once()
 
+    def test_invalid_toml_file(self):
+        result = self.runner.invoke(run, ["--config", str(self.invalid_config_path), "--loglevel", "INFO"])
+        self.assertNotEqual(result.exit_code, 0)
+        self.assertIn("Error decoding TOML", result.output)
+
 
 if __name__ == "__main__":
     unittest.main()