Merge pull request #337 from NOAA-GFDL/main

merging main in, before doing the reverse

.gitignore

@@ -66,6 +66,10 @@ fre/tests/test_files/outdirfre/
 fre/pp/tests/configure_yaml_out/
 fre/yamltools/tests/combine_yamls_out/
 fre/app/generate_time_averages/tests/time_avg_test_files/
+fre/make/tests/null_example/combined-null_model.yaml
+fre/make/tests/makefile_out
+fre/make/tests/compile_out
+fre/tests/fremake_out
 
 # Translations
 *.mo

README.md

@@ -86,5 +86,4 @@ To be developed:
 - [x]  **fre make**
 - [x]  **fre pp**
 - [ ]  **fre run**
-- [ ]  **fre test**
-- [x]  **fre yamltools**
+- [x]  **fre yamltools**

fre/fre.py

@@ -31,7 +31,6 @@
                        "list": ".list.frelist.list_cli",
                        "check": ".check.frecheck.check_cli",
                        "run": ".run.frerun.run_cli",
-                       "test": ".test.fretest.test_cli",
                        "yamltools": ".yamltools.freyamltools.yamltools_cli",
                        "make": ".make.fremake.make_cli",
                        "app": ".app.freapp.app_cli",

fre/make/create_compile_script.py

@@ -16,7 +16,6 @@ def compile_create(yamlfile, platform, target, jobs, parallel, execute, verbose)
     name = yamlfile.split(".")[0]
     nparallel = parallel
     jobs = str(jobs)
-    run = execute
 
     if verbose:
       logging.basicCOnfig(level=logging.INFO)
@@ -80,13 +79,14 @@ def compile_create(yamlfile, platform, target, jobs, parallel, execute, verbose)
                    fremakeBuild.writeBuildComponents(c)
               fremakeBuild.writeScript()
               fremakeBuildList.append(fremakeBuild)
-              print("\nCompile script created at " + bldDir + "/compile.sh" + "\n") #was click.echo
-    if run:
+              print("\nCompile script created at " + bldDir + "/compile.sh" + "\n")
+
+    if execute:
         if baremetalRun:
             pool = Pool(processes=nparallel)                         # Create a multiprocessing Pool
             pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)  # process data_inputs iterable with pool
     else:
-        return #0 #sys.exit()
+        return
 
 if __name__ == "__main__":
     compile_create()

fre/make/create_docker_script.py

@@ -52,14 +52,16 @@ def dockerfile_create(yamlfile, platform, target, execute):
             ## Check for type of build
             if platform["container"] is True:
                 image=modelYaml.platforms.getContainerImage(platformName)
+                stage2image = modelYaml.platforms.getContainer2base(platformName)
                 bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
                 tmpDir = "tmp/"+platformName
                 dockerBuild = buildDocker.container(base = image,
                                               exp = fremakeYaml["experiment"],
                                               libs = fremakeYaml["container_addlibs"],
                                               RUNenv = platform["RUNenv"],
                                               target = targetObject,
-                                              mkTemplate = platform["mkTemplate"])
+                                              mkTemplate = platform["mkTemplate"],
+                                              stage2base = stage2image)
                 dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
                 dockerBuild.writeDockerfileMakefile(tmpDir+"/Makefile", tmpDir+"/linkline.sh")
 

fre/make/gfdlfremake/buildDocker.py

@@ -18,8 +18,12 @@ class container():
         - RUNenv : The commands that have to be run at
                    the beginning of a RUN in the dockerfile
                    to set up the environment
+        - target : The FRE target
+        - mkTemplate: The mkmf template to use
+        - stage2base: The base for the second stage. Empty
+                      string if there is no second stage
     """
-    def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
+    def __init__(self,base,exp,libs,RUNenv,target,mkTemplate,stage2base):
         """
         Initialize variables and write to the dockerfile
         """
@@ -31,6 +35,7 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
         self.mkmf = True
         self.target = target
         self.template = mkTemplate
+        self.stage2base = stage2base
 
         # Set up spack loads in RUN commands in dockerfile
         if RUNenv == "":
@@ -54,15 +59,15 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
                        " && src_dir="+self.src+" \\ \n",
                        " && mkmf_template="+self.template+ " \\ \n"]
         self.d=open("Dockerfile","w")
-        self.d.writelines("FROM "+self.base+" \n")
-        if self.base == "ecpe4s/noaa-intel-prototype:2023.09.25":
-            self.prebuild = '''RUN 
-            '''
-            self.postbuild = '''
-            '''
-            self.secondstage = '''
-            '''
-
+        self.d.writelines("FROM "+self.base+" as builder\n")
+        ## Set up the second stage build list of lines to add
+        if self.stage2base == "":
+            self.secondstage = ["\n"]
+        else:
+            self.secondstage = [f"FROM {self.stage2base} as final\n",
+                                f"COPY --from=builder  {self.src} {self.src}\n",
+                                f"COPY --from=builder {self.bld} {self.bld}\n",
+                                f"ENV PATH=$PATH:{self.bld}\n"]
     def writeDockerfileCheckout(self, cScriptName, cOnDisk):
         """
         Brief: writes to the checkout part of the Dockerfile and sets up the compile
@@ -170,6 +175,9 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk):
                                  "export BACKUP_LD_LIBRARY_PATH=$LD_LIBRARY_PATH\n",
                                  "# Set up spack loads\n",
                                  RUNenv[0]+"\n"]
+            #create directory if not present
+            os.makedirs(os.path.dirname(runOnDisk), exist_ok=True)
+            # write file
             with open(runOnDisk,"w") as f:
                 f.writelines(self.createscript)
                 f.write("# Load spack packages\n")
@@ -201,6 +209,9 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk):
             self.d.write(" cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n")
         else:
             self.d.write(" && cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n")
+        ## Write any second stage lines here 
+        for l in self.secondstage:
+            self.d.write(l)
         self.d.write('ENTRYPOINT ["/bin/bash"]')
         self.d.close()
 

fre/make/gfdlfremake/platformfre.py

@@ -41,36 +41,49 @@ def __init__(self,platforminfo):
             ## Check if we are working with a container and get the info for that
             try:
                 p["container"]
+            ## When not doing a container build, this should all be set to empty strings and Falses
             except:
                 p["container"] = False
                 p["RUNenv"] = [""]
                 p["containerBuild"] = ""
                 p["containerRun"] = ""
-                p["containerViews"] = False
                 p["containerBase"] = ""
-                p["container2step"] = ""
+                p["container2step"] = False
+                p["container2base"] = ""
             if p["container"]:
                 ## Check the container builder
                 try:
                     p["containerBuild"]
                 except:
-                    raise Exception("You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n")
+                    raise Exception("Platform "+p["name"]+": You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n")
                 if p["containerBuild"] != "podman" and p["containerBuild"] != "docker":
-                    raise ValueError("Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n")
-                print (p["containerBuild"])
-## Check for container environment set up for RUN commands
+                    raise ValueError("Platform "+p["name"]+": Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n")
+                ## Get the name of the base container
                 try:
                     p["containerBase"]
-                except NameError:
-                    print("You must specify the base container you wish to use to build your application")
-                try:
-                    p["containerViews"]
                 except:
-                    p["containerViews"] = False
+                    raise NameError("Platform "+p["name"]+": You must specify the base container you wish to use to build your application")
+                ## Check if this is a 2 step (multi stage) build
                 try:
                     p["container2step"]
                 except:
-                    p["container2step"] = ""
+                    p["container2step"] = False
+                ## Get the base for the second stage of the build
+                if p["container2step"]:
+                    try:
+                        p["container2base"]
+                    except:
+                        raise NameError ("Platform "+p["name"]+": container2step is True, so you must define a container2base\n")
+                    ## Check if there is anything special to copy over
+                else:
+                    ## There should not be a second base if this is not a 2 step build
+                    try:
+                        p["container2base"]
+                    except:
+                        p["container2base"] = ""
+                    else:
+                        raise ValueError ("Platform "+p["name"]+": You defined container2base "+p["container2base"]+" but container2step is False\n")
+                ## Get any commands to execute in the dockerfile RUN command
                 try:
                     p["RUNenv"]
                 except:
@@ -83,6 +96,7 @@ def __init__(self,platforminfo):
                 if p["containerRun"] != "apptainer" and p["containerRun"] != "singularity":
                     raise ValueError("Container builds only supported with apptainer, but you listed "+p["containerRun"]+"\n")
             else:
+                ## Find the location of the mkmf template
                 try:
                     p["mkTemplate"]
                 except:
@@ -120,7 +134,6 @@ def getContainerInfoFromName(self,name):
                 p["RUNenv"], \
                 p["containerBuild"], \
                 p["containerRun"], \
-                p["containerViews"], \
                 p["containerBase"], \
                 p["container2step"])
     def isContainer(self, name):
@@ -137,3 +150,10 @@ def getContainerImage(self,name):
         for p in self.yaml:
             if p["name"] == name:
                 return p["containerBase"]
+    def getContainer2base(self,name):
+        """
+        Brief: returns the image to be used in the second step of the Dockerfile
+        """
+        for p in self.yaml:
+            if p["name"] == name:
+                return p["container2base"]

fre/make/gfdlfremake/yamlfre.py

@@ -171,7 +171,6 @@ def __init__(self,combinedyaml,v):
 
         #get platform info
         self.platformsdict = self.freyaml.get("platforms")
-        print(self.platformsdict)
         self.platforms = platformfre.platforms(self.platformsdict)
         self.platformsyaml = self.platforms.getPlatformsYaml()
 

fre/make/run_fremake_script.py

@@ -147,6 +147,7 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
                 ###################### container stuff below #######################################
                 ## Run the checkout script
                 image=modelYaml.platforms.getContainerImage(platformName)
+                stage2image = modelYaml.platforms.getContainer2base(platformName)
                 srcDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src"
                 bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
                 tmpDir = "tmp/"+platformName
@@ -174,7 +175,8 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
                                                     libs = fremakeYaml["container_addlibs"],
                                                     RUNenv = platform["RUNenv"],
                                                     target = target,
-                                                    mkTemplate = platform["mkTemplate"])
+                                                    mkTemplate = platform["mkTemplate"],
+                                                    stage2base = stage2image)
                 dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
                 dockerBuild.writeDockerfileMakefile(freMakefile.getTmpDir() + "/Makefile",
                                                     freMakefile.getTmpDir() + "/linkline.sh")