Merge pull request #226 from IBM/fix/223-support-for-rules-to-run-sys…

…tem-commands

Fix/223 support for rules to run system commands and Fix 225 support for splitting lines with escaped newlines

CHANGELOG

@@ -1,3 +1,8 @@
+* Thu Apr 27 2023 Edmund Reinhardt <[email protected]> - 2.4.12
+- Support rules with no dependencies, which is useful for custom recipes.
+  Addresses https://github.com/IBM/ibmi-bob/issues/223
+- Support the splitting of lines using escaped newlines as described by
+  https://www.gnu.org/software/make/manual/make.html#Splitting-Lines
 * Mon Apr 17 2023 Edmund Reinhardt <[email protected]> - 2.4.11
 - Fix SAVOBJ/RSTOBJ command for crtfrmstmf - failed when deleting files that
   depend on a PF. https://github.com/IBM/ibmi-bob/issues/215 and 

src/makei/rules_mk.py

@@ -115,8 +115,9 @@ def from_str(rule_str: str, containing_dir: Path, include_dirs: List[Path]) -> "
         >>> str(rule)
         'target : dependency1 dependency2\n\tcommand1 param1 param2\n\tcommand2 param3 param4\n'
         """
-        rule_str = rule_str.strip()
-        rule_regex = re.compile(r"^(?P<target>\S+)\s*:(?!=)\s*(?P<dependencies>\S+.*)(?P<cmds>(\n[^\S\r\n]+\S.*)*)$")
+        rule_regex = re.compile(
+            r"^(?P<target>\S+)[ \t]*:(?!=)[ \t]*(?P<dependencies>(?:[^\n]*)*)\n" +
+            r"(?P<cmds>(?:[^\S\r\n]+?\S[^\n]*\n?|\s*\n)*)$")
         target_match = rule_regex.match(rule_str)
         if target_match:
             target = target_match.group("target")
@@ -126,7 +127,7 @@ def from_str(rule_str: str, containing_dir: Path, include_dirs: List[Path]) -> "
         else:
             raise ValueError(f"Invalid rule string '{rule_str}'")
 
-        return MKRule(target, dependencies, commands, {}, containing_dir, include_dirs)
+        return MKRule(target, dependencies, commands, [], containing_dir, include_dirs)
 
 
 class RulesMk:
@@ -184,7 +185,7 @@ def from_str(cls, rules_mk_str: str, containing_dir: Path, include_dirs=None) ->
 
         if include_dirs is None:
             include_dirs = []
-        rules_mk_str = rules_mk_str.strip()
+        rules_mk_str = rules_mk_str.strip().replace("\\\n", "")
 
         rules = []
         variables = {}

tests/data/rules_mks/custom.rules.mk

@@ -0,0 +1,2 @@
+CRTSBSD.FILE:
+	system -i "CRTSBSD SBSD(BATCHWL/BATCHSBSD) POOLS((1 *SHRPOOL3 *N *MB))"

tests/data/rules_mks/dds.rules.mk

@@ -0,0 +1,12 @@
+ARTICLE.FILE:  ARTICLE-Article_File.PF \
+  SAMREF.FILE
+ART301D.FILE:  ART301D-Function_Select_an_article.DSPF\
+  ARTICLE.FILE VATDEF.FILE
+
+
+DETORD.FILE:  DETORD.PF SAMREF.FILE
+ORD500O.FILE: ORD500O.PRTF ORDER.FILE CUSTOMER.FILE \
+  DETORD.FILE ARTICLE.FILE
+TMPDETORD.FILE: 	
+ 	system -i "CPYF FROMFILE($(OBJLIB)/DETORD) TOFILE($(OBJLIB)/TMPDETORD) \ 
+	CRTFILE(*YES)"

tests/unit/test_rules_mk.py

@@ -47,3 +47,99 @@ def test_from_file():
 VAT300.MODULE: private VARIMMED ::= IMMED
 VAT300.MODULE: private VARESCAPE :::= ESCAPE
 '''
+
+
+def test_custom_recipe():
+    # Test loading from a valid file
+    rules_mk = RulesMk.from_file(data_dir / "custom.rules.mk")
+    expected_targets = {'TRGs': [], 'DTAs': [], 'SQLs': [], 'BNDDs': [], 'PFs': ['CRTSBSD.FILE'], 'LFs': [],
+                        'DSPFs': [], 'PRTFs': [], 'CMDs': [], 'MODULEs': [], 'SRVPGMs': [], 'PGMs': [],
+                        'MENUs': [], 'PNLGRPs': [], 'QMQRYs': [], 'WSCSTs': [], 'MSGs': []}
+    commands0 = ['@$(call echo_cmd,=== Creating [CRTSBSD.FILE] from custom recipe)',
+                 'system -i "CRTSBSD SBSD(BATCHWL/BATCHSBSD) POOLS((1 *SHRPOOL3 *N *MB))"',
+                 '@$(call echo_success_cmd,End of creating CRTSBSD.FILE)']
+    assert rules_mk.containing_dir == data_dir
+    assert rules_mk.subdirs == []
+    assert rules_mk.targets == expected_targets
+    assert rules_mk.rules[0].variables == []
+    assert rules_mk.rules[0].commands == commands0
+    assert rules_mk.rules[0].dependencies == []
+    assert rules_mk.rules[0].include_dirs == []
+    assert rules_mk.rules[0].target == 'CRTSBSD.FILE'
+    assert rules_mk.rules[0].source_file is None
+    assert rules_mk.build_context is None
+    assert str(rules_mk.rules[0]) == '''CRTSBSD.FILE_CUSTOM_RECIPE=true
+CRTSBSD.FILE : \n\t@$(call echo_cmd,=== Creating [CRTSBSD.FILE] from custom recipe)
+\tsystem -i "CRTSBSD SBSD(BATCHWL/BATCHSBSD) POOLS((1 *SHRPOOL3 *N *MB))"
+\t@$(call echo_success_cmd,End of creating CRTSBSD.FILE)
+'''
+    assert str(rules_mk) == '''PFs := CRTSBSD.FILE
+
+
+CRTSBSD.FILE_CUSTOM_RECIPE=true
+CRTSBSD.FILE : \n\t@$(call echo_cmd,=== Creating [CRTSBSD.FILE] from custom recipe)
+\tsystem -i "CRTSBSD SBSD(BATCHWL/BATCHSBSD) POOLS((1 *SHRPOOL3 *N *MB))"
+\t@$(call echo_success_cmd,End of creating CRTSBSD.FILE)
+'''
+
+
+def test_dds_recipe():
+    # Test loading from a valid file
+    rules_mk = RulesMk.from_file(data_dir / "dds.rules.mk")
+    expected_targets = {'TRGs': [], 'DTAs': [], 'SQLs': [], 'BNDDs': [], 'PFs': ['ARTICLE.FILE',
+                        'DETORD.FILE', 'TMPDETORD.FILE'], 'LFs': [], 'DSPFs': ['ART301D.FILE'],
+                        'PRTFs': ['ORD500O.FILE'], 'CMDs': [], 'MODULEs': [], 'SRVPGMs': [], 'PGMs': [],
+                        'MENUs': [], 'PNLGRPs': [], 'QMQRYs': [], 'WSCSTs': [], 'MSGs': []}
+
+    assert rules_mk.containing_dir == data_dir
+    assert rules_mk.subdirs == []
+    assert rules_mk.targets == expected_targets
+    assert rules_mk.rules[0].variables == []
+    assert rules_mk.rules[0].commands == []
+    assert rules_mk.rules[0].dependencies == ['SAMREF.FILE']
+    assert rules_mk.rules[0].include_dirs == []
+    assert rules_mk.rules[0].target == 'ARTICLE.FILE'
+    assert rules_mk.rules[0].source_file == 'ARTICLE-Article_File.PF'
+    assert str(rules_mk.rules[0]) == '''ARTICLE.FILE_SRC=ARTICLE-Article_File.PF\nARTICLE.FILE_DEP=SAMREF.FILE
+ARTICLE.FILE_RECIPE=PF_TO_FILE_RECIPE\n'''
+
+    assert rules_mk.rules[1].variables == []
+    assert rules_mk.rules[1].commands == []
+    assert rules_mk.rules[1].dependencies == ['ARTICLE.FILE', 'VATDEF.FILE']
+    assert rules_mk.rules[1].include_dirs == []
+    assert rules_mk.rules[1].target == 'ART301D.FILE'
+    assert rules_mk.rules[1].source_file == 'ART301D-Function_Select_an_article.DSPF'
+    assert str(rules_mk.rules[1]) == '''ART301D.FILE_SRC=ART301D-Function_Select_an_article.DSPF
+ART301D.FILE_DEP=ARTICLE.FILE VATDEF.FILE\nART301D.FILE_RECIPE=DSPF_TO_FILE_RECIPE\n'''
+
+    assert rules_mk.rules[4].variables == []
+    assert rules_mk.rules[4].commands == ['@$(call echo_cmd,=== Creating [TMPDETORD.FILE] from custom recipe)',
+                                          'system -i "CPYF FROMFILE($(OBJLIB)/DETORD) TOFILE($(OBJLIB)/TMPDETORD) \\',
+                                          'CRTFILE(*YES)"',
+                                          '@$(call echo_success_cmd,End of creating TMPDETORD.FILE)']
+    assert rules_mk.rules[4].dependencies == []
+    assert rules_mk.rules[4].include_dirs == []
+    assert rules_mk.rules[4].target == 'TMPDETORD.FILE'
+    assert rules_mk.rules[4].source_file is None
+    assert str(rules_mk.rules[4]) == '''TMPDETORD.FILE_CUSTOM_RECIPE=true
+TMPDETORD.FILE : \n\t@$(call echo_cmd,=== Creating [TMPDETORD.FILE] from custom recipe)
+\tsystem -i "CPYF FROMFILE($(OBJLIB)/DETORD) TOFILE($(OBJLIB)/TMPDETORD) \\
+\tCRTFILE(*YES)"\n\t@$(call echo_success_cmd,End of creating TMPDETORD.FILE)\n'''
+
+    assert str(rules_mk) == '''PFs := ARTICLE.FILE DETORD.FILE TMPDETORD.FILE
+DSPFs := ART301D.FILE
+PRTFs := ORD500O.FILE\n\n
+ARTICLE.FILE_SRC=ARTICLE-Article_File.PF\nARTICLE.FILE_DEP=SAMREF.FILE
+ARTICLE.FILE_RECIPE=PF_TO_FILE_RECIPE
+ART301D.FILE_SRC=ART301D-Function_Select_an_article.DSPF
+ART301D.FILE_DEP=ARTICLE.FILE VATDEF.FILE
+ART301D.FILE_RECIPE=DSPF_TO_FILE_RECIPE\nDETORD.FILE_SRC=DETORD.PF
+DETORD.FILE_DEP=SAMREF.FILE
+DETORD.FILE_RECIPE=PF_TO_FILE_RECIPE\nORD500O.FILE_SRC=ORD500O.PRTF
+ORD500O.FILE_DEP=ORDER.FILE CUSTOMER.FILE DETORD.FILE ARTICLE.FILE
+ORD500O.FILE_RECIPE=PRTF_TO_FILE_RECIPE\nTMPDETORD.FILE_CUSTOM_RECIPE=true
+TMPDETORD.FILE : \n\t@$(call echo_cmd,=== Creating [TMPDETORD.FILE] from custom recipe)
+\tsystem -i "CPYF FROMFILE($(OBJLIB)/DETORD) TOFILE($(OBJLIB)/TMPDETORD) \\
+\tCRTFILE(*YES)"
+\t@$(call echo_success_cmd,End of creating TMPDETORD.FILE)
+'''