Py3 port branch

config/cesm/machines/config_machines.xml

@@ -214,7 +214,7 @@
     <BATCH_SYSTEM>none</BATCH_SYSTEM>
     <SUPPORTED_BY>[email protected]</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>8</MAX_TASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE>4</MAX_MPITASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>8</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpiexec</executable>

scripts/Tools/case.build

@@ -84,7 +84,7 @@ def _main_func(description):
         if cleanlist is not None or clean_all:
             build.clean(case, cleanlist, clean_all)
         elif(testname is not None):
-            logging.warn("Building test for {} in directory {}".format(testname,
+            logging.warning("Building test for {} in directory {}".format(testname,
                                                                        caseroot))
             try:
                 # The following line can throw exceptions if the testname is

scripts/Tools/case_diff

@@ -68,8 +68,8 @@ def recursive_diff(dir1, dir2, repls, show_binary=False, skip_list=()):
     for dirname, set_obj in [(dir1, dir1_only), (dir2, dir2_only)]:
         for item in sorted(set_obj):
             if (item not in skip_list):
-                print "==============================================================================="
-                print os.path.join(dirname, item), "is unique"
+                print ("===============================================================================")
+                print (os.path.join(dirname, item), "is unique")
                 num_differing_files += 1
 
     # Handling of the common items is trickier
@@ -82,8 +82,8 @@ def recursive_diff(dir1, dir2, repls, show_binary=False, skip_list=()):
 
         # If the directory status of the files differs, report diff
         if (path1isdir != os.path.isdir(path2)):
-            print "==============================================================================="
-            print path1 + " DIFFERS (directory status)"
+            print ("===============================================================================")
+            print (path1 + " DIFFERS (directory status)")
             num_differing_files += 1
             continue
 
@@ -100,7 +100,7 @@ def recursive_diff(dir1, dir2, repls, show_binary=False, skip_list=()):
 
             stat, out, err = run_cmd("file {}".format(path1))
             if (stat != 0):
-                logging.warn("Failed to probe file '{}', out: '{}', err: '{}'".format(path1, out, err))
+                logging.warning("Failed to probe file '{}', out: '{}', err: '{}'".format(path1, out, err))
                 continue
 
             is_text_file = "text" in out
@@ -111,10 +111,10 @@ def recursive_diff(dir1, dir2, repls, show_binary=False, skip_list=()):
 
                 stat, out, _ = run_cmd("diff -w {} -".format(path1), input_str=the_text)
                 if (stat != 0):
-                    print "==============================================================================="
-                    print path1 + " DIFFERS (contents)"
+                    print ("===============================================================================")
+                    print (path1 + " DIFFERS (contents)")
                     num_differing_files += 1
-                    print "  ", out
+                    print ("  "+ out)
 
     return num_differing_files
 
@@ -135,7 +135,7 @@ def _main_func(description):
         repls[val2] = val1
 
     num_differing_files = recursive_diff(case1, case2, repls, show_binary, skip_list)
-    print num_differing_files, "files are different"
+    print (num_differing_files, "files are different")
     sys.exit(0 if num_differing_files == 0 else 1)
 
 ###############################################################################

scripts/Tools/cime_bisect

@@ -95,9 +95,9 @@ def cime_bisect(testargs, good, bad, testroot, compiler, project, baseline_name,
         commits_we_want_to_test = run_cmd_no_fail("git rev-list {}..{} --merges --first-parent".format(good, bad)).splitlines()
         all_commits_            = run_cmd_no_fail("git rev-list {}..{}".format(good, bad)).splitlines()
         commits_to_skip         = set(all_commits_) - set(commits_we_want_to_test)
-        print "Skipping these non-merge commits"
+        print("Skipping these non-merge commits")
         for item in commits_to_skip:
-            print item
+            print(item)
     else:
         commits_to_skip = set()
 
@@ -145,8 +145,8 @@ def cime_bisect(testargs, good, bad, testroot, compiler, project, baseline_name,
         bad_commits_filtered = bad_commits - commits_to_skip
 
         expect(len(bad_commits_filtered) == 1, bad_commits_filtered)
-        print "Bad merge is:"
-        print run_cmd_no_fail("git show {}".format(bad_commits_filtered.pop()))
+        print("Bad merge is:")
+        print(run_cmd_no_fail("git show {}".format(bad_commits_filtered.pop())))
 
     finally:
         run_cmd_no_fail("git bisect reset")

scripts/Tools/cimeteststatus

@@ -5,9 +5,9 @@ Purpose:  Give basic and detailed summaries of CIME(CESM) tests,
 and send the test results back to the test database.
 Authors: Jay Shollenberger and Ben Andre
 """
-
-from __future__ import print_function
-import sys
+from standard_script_setup import *
+#pylint: disable=import-error
+from six.moves import urllib
 if sys.hexversion < 0x02070000:
     print(70 * "*")
     print("ERROR: {0} requires python >= 2.7.x. ".format(sys.argv[0]))
@@ -17,12 +17,11 @@ if sys.hexversion < 0x02070000:
     sys.exit(1)
 import xml.etree.ElementTree as etree
 import argparse
-import os, glob, re
-import urllib
-import urllib2
+import glob, re
 import pprint
 import getpass
 
+
 testdburl = "https://csegweb.cgd.ucar.edu/testdb/cgi-bin/processXMLtest.cgi"
 
 class CimeTestStatus():
@@ -354,10 +353,10 @@ def getSuiteInfo(specfile):
 
     for t in root.findall('test'):
         testlist.append(t.get('case'))
-        if 'machine' not in suiteinfo.keys():
+        if 'machine' not in suiteinfo:
             machnodelist = t.findall('mach')
             suiteinfo['machine'] = machnodelist[0].text
-        if 'compiler' not in suiteinfo.keys():
+        if 'compiler' not in suiteinfo:
             compnodelist = t.findall('compiler')
             suiteinfo['compiler'] = compnodelist[0].text
 
@@ -527,13 +526,13 @@ def sendTestReport(args, suiteinfo, cimetests, auth):
     # Get the testdb username/password, and POST
     # the data.
     print("sending test report for " + suiteinfo['machine'] + " " + suiteinfo['compiler'])
-    data = urllib.urlencode({'username':auth['username'],
+    data = urllib.parse.urlencode({'username':auth['username'],
                              'password':auth['password'],
                              'testXML':testrecord})
-    req = urllib2.Request(testdburl, data)
+    req = urllib.request.Request(testdburl, data)
     try:
-        urllib2.urlopen(req)
-    except urllib2.URLError as e:
+        urllib.request.urlopen(req)
+    except urllib.error.URLError as e:
         print("Error when posting data: " + e.reason)
 
     if(args.debug):
@@ -552,7 +551,7 @@ def authenticate():
     auth['password'] = getpass.getpass("enter TestDB password:")
     return auth
 
-def main():
+def _main_func():
     """ Parse the arguments, get the suite information from the test spec,
     get the test statuses, then print a raw status, test summary, or send the
     test report.
@@ -596,5 +595,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
-
+    _main_func()

scripts/Tools/code_checker

@@ -63,7 +63,7 @@ def _main_func(description):
     num_procs, files = parse_command_line(sys.argv, description)
 
     results = check_code(files, num_procs=num_procs, interactive=True)
-    for result in results.itervalues():
+    for result in results.values():
         if result != "":
             sys.exit(1)
 

scripts/Tools/compare_namelists

@@ -67,7 +67,7 @@ def _main_func(description):
     expect(success,
            "Namelist diff between files {} and {}\n{}".format(gold_file, compare_file, comments))
 
-    print "Files {} and {} MATCH".format(gold_file, compare_file)
+    print("Files {} and {} MATCH".format(gold_file, compare_file))
 
 ###############################################################################
 

scripts/Tools/component_compare_baseline

@@ -45,7 +45,7 @@ def _main_func(description):
     caseroot, baseline_dir = parse_command_line(sys.argv, description)
     with Case(caseroot) as case:
         success, comments = compare_baseline(case, baseline_dir)
-        print comments
+        print(comments)
 
     sys.exit(0 if success else 1)
 

scripts/Tools/component_compare_test

@@ -48,7 +48,7 @@ def _main_func(description):
     suffix1, suffix2, caseroot = parse_command_line(sys.argv, description)
     with Case(caseroot) as case:
         success, comments = compare_test(case, suffix1, suffix2)
-        print comments
+        print(comments)
 
     sys.exit(0 if success else 1)
 

scripts/Tools/component_generate_baseline

@@ -52,7 +52,7 @@ def _main_func(description):
     with Case(caseroot) as case:
         success, comments = generate_baseline(case, baseline_dir,
                                               allow_baseline_overwrite)
-        print comments
+        print(comments)
 
     sys.exit(0 if success else 1)
 

scripts/Tools/cs.status

@@ -65,8 +65,9 @@ def cs_status(test_paths, summary=False):
             test_id_output[test_id] = output
 
     for test_id in sorted(test_id_output):
-        print test_id
-        print test_id_output[test_id],
+        print(test_id)
+        print(test_id_output[test_id])
+        print(' ')
 
 ###############################################################################
 def _main_func(description):

scripts/Tools/pelayout

@@ -113,10 +113,10 @@ def print_pelayout(case, ntasks, nthreads, rootpes, arg_format, header):
     comp_classes = case.get_values("COMP_CLASSES")
 
     if (header is not None):
-        print header
+        print(header)
     # End if
     for comp in comp_classes:
-        print format_pelayout(comp, ntasks[comp], nthreads[comp], rootpes[comp], arg_format)
+        print(format_pelayout(comp, ntasks[comp], nthreads[comp], rootpes[comp], arg_format))
     # End for
 
 # End def print_pelayout

scripts/Tools/preview_run

@@ -57,13 +57,13 @@ def _main_func(description):
     logging.disable(logging.CRITICAL)
 
     with Case(caseroot, read_only=False) as case:
-        print "BATCH SUBMIT:"
+        print("BATCH SUBMIT:")
         job = "case.test" if case.get_value("TEST") else "case.run"
         job_id_to_cmd = case.submit_jobs(dry_run=True, job=job)
         for job_id, cmd in job_id_to_cmd:
-            print "   ", job_id, "->", case.get_resolved_value(cmd)
-        print
-        print "MPIRUN:", case.get_resolved_value(case.get_mpirun_cmd())
+            print("   ", job_id, "->", case.get_resolved_value(cmd))
+        print()
+        print("MPIRUN:", case.get_resolved_value(case.get_mpirun_cmd()))
 
 if __name__ == "__main__":
     _main_func(__doc__)

scripts/Tools/xmlquery

@@ -327,38 +327,38 @@ def _main_func():
         wrapper.fix_sentence_endings = True
 
     for group in sorted(iter(results)):
-        if (len(variables) > 1 or len(results.keys()) > 1 or full) and not get_group:
-            print "\nResults in group %s"%group
+        if (len(variables) > 1 or len(results) > 1 or full) and not get_group:
+            print("\nResults in group %s"%group)
         for var in variables:
             if var in results[group]:
                 if raw:
-                    print results[group][var]['raw']
+                    print(results[group][var]['raw'])
                 elif get_group:
-                    print "\t%s: %s"%(var, results[group][var]['get_group'])
+                    print("\t%s: %s"%(var, results[group][var]['get_group']))
                 elif value:
                     sys.stdout.write("%s"%results[group][var]['value'])
                 elif description:
                     if results[group][var]['desc'][0] is not None:
                         desc_text = ' '.join(results[group][var]['desc'][0].split())
-                        print "\t%s: %s"%(var, wrapper.fill(desc_text))
+                        print("\t%s: %s"%(var, wrapper.fill(desc_text)))
                 elif fileonly:
-                    print "\t%s: %s"%(var, results[group][var]['file'])
+                    print("\t%s: %s"%(var, results[group][var]['file']))
                 elif dtype:
-                    print  "\t%s: %s"%(var, results[group][var]['type'])
+                    print("\t%s: %s"%(var, results[group][var]['type']))
                 elif valid_values:
                     if 'valid_values' in results[group][var]:
-                        print  "\t%s: %s"%(var, results[group][var]["valid_values"])
+                        print("\t%s: %s"%(var, results[group][var]["valid_values"]))
                 elif full:
                     if results[group][var]['desc'][0] is not None:
                         desc_text = ' '.join(results[group][var]['desc'][0].split())
-                    print "\t%s: value=%s"%(var, results[group][var]['value'])
-                    print  "\t\ttype: %s"%(results[group][var]['type'][0])
+                    print("\t%s: value=%s"%(var, results[group][var]['value']))
+                    print("\t\ttype: %s"%(results[group][var]['type'][0]))
                     if 'valid_values' in results[group][var]:
-                        print  "\t\tvalid_values: %s"%(results[group][var]["valid_values"])
-                    print "\t\tdescription: %s"%(wrapper.fill(desc_text))
-                    print "\t\tfile: %s"%(results[group][var]['file'][0])
+                        print("\t\tvalid_values: %s"%(results[group][var]["valid_values"]))
+                    print("\t\tdescription: %s"%(wrapper.fill(desc_text)))
+                    print("\t\tfile: %s"%(results[group][var]['file'][0]))
                 else:
-                    print "\t%s: %s"%(var, results[group][var]['value'])
+                    print("\t%s: %s"%(var, results[group][var]['value']))
 
 
 if (__name__ == "__main__"):

scripts/lib/CIME/BuildTools/configure.py

@@ -73,7 +73,7 @@ def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
     """
     ems_path = os.path.join(output_dir, "env_mach_specific.xml")
     if os.path.exists(ems_path):
-        logger.warn("{} already exists, delete to replace".format(ems_path))
+        logger.warning("{} already exists, delete to replace".format(ems_path))
         return
     ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing)
     ems_file.populate(machobj)