[SYCLomatic][BUG fix]Bug fixes for Python script migration (#2530)

clang/lib/DPCT/UserDefinedRules/PatternRewriter.cpp

@@ -271,7 +271,8 @@ static int parseCodeElement(const MatchPattern &Suffix,
   const int Size = Input.size();
   while (Index >= 0 && Index < Size) {
 
-    if (SrcFileType == SourceFileType::SFT_CMakeScript) {
+    if (SrcFileType == SourceFileType::SFT_CMakeScript ||
+        SrcFileType == SourceFileType::SFT_PySetupScript) {
       if (Input[Index] == '#') {
         for (; Index < Size && Input[Index] != '\n'; Index++) {
         }
@@ -280,8 +281,10 @@ static int parseCodeElement(const MatchPattern &Suffix,
     }
 
     const auto Character = Input[Index];
-    if(Suffix.size() == 0 && Character =='"') {
-      return Index;
+    if (SrcFileType != SourceFileType::SFT_PySetupScript) {
+      if (Suffix.size() == 0 && Character == '"') {
+        return Index;
+      }
     }
     if (Suffix.size() > 0) {
       std::optional<MatchResult> SuffixMatch;
@@ -322,7 +325,8 @@ static int parseCodeElement(const MatchPattern &Suffix,
     delimiters.
     */
 
-    if (SrcFileType == SourceFileType::SFT_CMakeScript) {
+    if (SrcFileType == SourceFileType::SFT_CMakeScript ||
+        SrcFileType == SourceFileType::SFT_PySetupScript) {
       if (Index - 1 >= 0 && Character == '"' && Input[Index - 1] == '\\') {
         Index++;
         while (Index < Size &&
@@ -414,6 +418,8 @@ static void applyExtenstionNameChange(
   for (; Pos > 0 && !isWhitespace(Input[Pos]); Pos--) {
   }
   Pos = Pos == 0 ? 0 : Pos + 1;
+  if (Input[Pos] == '"' || Input[Pos] == '\'')
+    Pos += 1;
   std::string SrcFile = Input.substr(Pos, Next + ExtensionType.length() +
                                               1 /*strlen of "."*/ - Pos);
   bool HasCudaSyntax = false;
@@ -563,7 +569,8 @@ findMatch(const MatchPattern &Pattern, const std::string &Input,
   const int Size = Input.size();
   while (PatternIndex < PatternSize && Index < Size) {
 
-    if (SrcFileType == SourceFileType::SFT_CMakeScript) {
+    if (SrcFileType == SourceFileType::SFT_CMakeScript ||
+        SrcFileType == SourceFileType::SFT_PySetupScript) {
       if (Input[Index] == '#') {
         for (; Index < Size && Input[Index] != '\n'; Index++) {
         }
@@ -620,7 +627,8 @@ findMatch(const MatchPattern &Pattern, const std::string &Input,
       }
       std::string ElementContents = Input.substr(Index, Next - Index);
 
-      if (SrcFileType == SourceFileType::SFT_CMakeScript) {
+      if (SrcFileType == SourceFileType::SFT_CMakeScript ||
+          SrcFileType == SourceFileType::SFT_PySetupScript) {
         if (Code.Name == "empty" && !ElementContents.empty() &&
             ElementContents.find_first_not_of(' ') != std::string::npos) {
           // For reversed variable ${empty}, it should be empty string or string
@@ -723,8 +731,8 @@ bool fixLineEndings(const std::string &Input, std::string &Output) {
   return isCRLF;
 }
 
-bool skipCmakeComments(std::ostream &OutputStream, const std::string &Input,
-                       int &Index) {
+bool skipBuildScriptComments(std::ostream &OutputStream,
+                             const std::string &Input, int &Index) {
   const int Size = Input.size();
   bool CommentFound = false;
   if (Input[Index] == '#') {
@@ -820,8 +828,9 @@ std::string applyPatternRewriter(const MetaRuleObject::PatternRewriter &PP,
   int Index = 0;
   while (Index < Size) {
 
-    if (SrcFileType == SourceFileType::SFT_CMakeScript) {
-      if (skipCmakeComments(OutputStream, Input, Index)) {
+    if (SrcFileType == SourceFileType::SFT_CMakeScript ||
+        SrcFileType == SourceFileType::SFT_PySetupScript) {
+      if (skipBuildScriptComments(OutputStream, Input, Index)) {
         continue;
       }
     }
@@ -835,7 +844,8 @@ std::string applyPatternRewriter(const MetaRuleObject::PatternRewriter &PP,
         const auto &SubruleIterator = PP.Subrules.find(Name);
         if (SubruleIterator != PP.Subrules.end()) {
 
-          if (SrcFileType == SourceFileType::SFT_CMakeScript) {
+          if (SrcFileType == SourceFileType::SFT_CMakeScript ||
+              SrcFileType == SourceFileType::SFT_PySetupScript) {
             auto Pos = Input.find(Value);
             if (Pos != std::string::npos) {
               FrontPart = Input.substr(0, Pos);
@@ -849,7 +859,8 @@ std::string applyPatternRewriter(const MetaRuleObject::PatternRewriter &PP,
 
       std::string OutStr = PP.Out;
 
-      if (SrcFileType == SourceFileType::SFT_CMakeScript &&
+      if ((SrcFileType == SourceFileType::SFT_CMakeScript ||
+           SrcFileType == SourceFileType::SFT_PySetupScript) &&
           !PP.Warning.empty() && Result->FullMatchFound) {
         constructWaringMsg(Input, Match.End, FileName, FrontPart, PP.Warning,
                            OutStr);

clang/test/dpct/python_migration/case_004/expected.py

@@ -1 +1,13 @@
-cuda_sources = list(glob.glob(os.path.join(extensions_cuda_dir, "*.cpp")))
+cuda_sources = list(glob.glob(os.path.join(extensions_cuda_dir, "*.dp.cpp")))
+
+# srcs in normal args (single & double quotes)
+## single quote
+out = func('foo.cpp', 'bar.dp.cpp')
+## double quotes
+out = func("foo.cpp", "bar.dp.cpp")
+
+# srcs in list arg
+## single quote
+out = func(['foo.cpp', 'bar.dp.cpp'])
+## double quotes
+out = func(["foo.cpp", "bar.dp.cpp"])

clang/test/dpct/python_migration/case_004/input.py

@@ -1 +1,13 @@
 cuda_sources = list(glob.glob(os.path.join(extensions_cuda_dir, "*.cu")))
+
+# srcs in normal args (single & double quotes)
+## single quote
+out = func('foo.cpp', 'bar.cu')
+## double quotes
+out = func("foo.cpp", "bar.cu")
+
+# srcs in list arg
+## single quote
+out = func(['foo.cpp', 'bar.cu'])
+## double quotes
+out = func(["foo.cpp", "bar.cu"])

clang/test/dpct/python_migration/case_005/expected.py

@@ -1,7 +1,7 @@
 setup(
     name='quant_cuda',
     ext_modules=[cpp_extension.DPCPPExtension(
-        'quant_cuda', ['quant_cuda.cpp', 'quant_cuda_kernel.cpp']
+        'quant_cuda', ["quant_cuda.cpp", 'quant_cuda_kernel.dp.cpp']
     , include_dirs=cpp_extension.include_paths(),)],
     cmdclass={'build_ext': cpp_extension.DpcppBuildExtension}
 )

clang/test/dpct/python_migration/case_005/input.py

@@ -1,7 +1,7 @@
 setup(
     name='quant_cuda',
     ext_modules=[cpp_extension.CUDAExtension(
-        'quant_cuda', ['quant_cuda.cpp', 'quant_cuda_kernel.cu']
+        'quant_cuda', ["quant_cuda.cpp", 'quant_cuda_kernel.cu']
     )],
     cmdclass={'build_ext': cpp_extension.BuildExtension}
 )

clang/test/dpct/python_migration/case_007/MainSourceFiles.yaml

@@ -0,0 +1,83 @@
+---
+MainSourceFile:  '/path/to/MainSrcFiles_placehold'
+Replacements:
+  - FilePath:        '/path/to/src/foo.cpp'
+    Offset:          0
+    Length:          0
+    ReplacementText: ''
+    ConstantFlag:    ''
+    ConstantOffset:  0
+    InitStr:         ''
+    NewHostVarName:  ''
+    BlockLevelFormatFlag: false
+  - FilePath:        '/path/to/src/baz.cpp'
+    Offset:          0
+    Length:          0
+    ReplacementText: ''
+    ConstantFlag:    ''
+    ConstantOffset:  0
+    InitStr:         ''
+    NewHostVarName:  ''
+    BlockLevelFormatFlag: false
+
+MainSourceFilesDigest:
+  - MainSourceFile:  '/path/to/src/foo.cpp'
+    Digest:          e2636fb8d174ac319083b0306294d3bd
+    HasCUDASyntax:   true
+  - MainSourceFile:  '/path/to/src/baz.cpp'
+    Digest:          991d7e4825fc597205bf68f2eda27acd
+    HasCUDASyntax:   false
+DpctVersion:     19.0.0
+MainHelperFileName: ''
+USMLevel:        ''
+FeatureMap:      {}
+CompileTargets:  {}
+OptionMap:
+  AnalysisScopePath:
+    Value:           '/path/to//target'
+    Specified:       false
+  AsyncHandler:
+    Value:           'false'
+    Specified:       false
+  BuildScript:
+    Value:           '1'
+    Specified:       true
+  CodePinEnabled:
+    Value:           'false'
+    Specified:       false
+  CommentsEnabled:
+    Value:           'false'
+    Specified:       false
+  CompilationsDir:
+    Value:           '/path/to/build'
+    Specified:       true
+  CtadEnabled:
+    Value:           'false'
+    Specified:       false
+  EnablepProfiling:
+    Value:           'true'
+    Specified:       true
+  ExperimentalFlag:
+    Value:           '16384'
+    Specified:       true
+  ExplicitNamespace:
+    Value:           '20'
+    Specified:       false
+  ExtensionDDFlag:
+    Value:           '0'
+    Specified:       false
+  ExtensionDEFlag:
+    Value:           '4294967295'
+    Specified:       false
+  RuleFile:
+    Value:           ''
+    ValueVec:
+      - '/path/to/python_build_script_migration_rule_ipex.yaml'
+    Specified:       false
+  SyclNamedLambda:
+    Value:           'false'
+    Specified:       false
+  UsmLevel:
+    Value:           '1'
+    Specified:       false
+...

clang/test/dpct/python_migration/case_007/case_007_test_extension_name.cpp

@@ -0,0 +1,12 @@
+// RUN: rm -rf %T && mkdir -p %T
+// RUN: cd %T
+// RUN: mkdir -p out
+// RUN: cp %S/input.py ./input.py
+// RUN: cp %S/MainSourceFiles.yaml ./out/MainSourceFiles.yaml
+// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
+// RUN: echo "end" >> %T/diff.txt
+
+// CHECK: begin
+// CHECK-NEXT: end

clang/test/dpct/python_migration/case_007/expected.py

@@ -0,0 +1,4 @@
+# baz.cpp is a C++ file
+out = func("bar.dp.cpp", "baz.cpp")
+# foo.cpp is a C++ file with CUDA syntax
+out = func("foo.cpp.dp.cpp", "bar.dp.cpp")

clang/test/dpct/python_migration/case_007/input.py

@@ -0,0 +1,4 @@
+# baz.cpp is a C++ file
+out = func("bar.cu", "baz.cpp")
+# foo.cpp is a C++ file with CUDA syntax
+out = func("foo.cpp", "bar.cu")

clang/test/dpct/python_migration/case_008/case_008_comments.cpp

@@ -0,0 +1,10 @@
+// RUN: rm -rf %T && mkdir -p %T
+// RUN: cd %T
+// RUN: cp %S/input.py ./input.py
+// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
+// RUN: echo "end" >> %T/diff.txt
+
+// CHECK: begin
+// CHECK-NEXT: end

clang/test/dpct/python_migration/case_008/expected.py

@@ -0,0 +1,3 @@
+# import torch
+import torch
+import intel_extension_for_pytorch

clang/test/dpct/python_migration/case_008/input.py

@@ -0,0 +1,2 @@
+# import torch
+import torch

clang/tools/dpct/DpctOptRules/python_build_script_migration_rule_ipex.yaml

@@ -133,6 +133,19 @@
   In: BuildExtension
   Out: DpcppBuildExtension
 
+- Rule: rule_cpp_file
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: cpp_file
+  In: ${func_name}(${value})
+  Out: ${func_name}(${value})
+  Subrules:
+    value:
+      MatchMode: Full
+      In: ${arg}.cpp
+      Out: ${arg}.${rewrite_extention_name}
+
 - Rule: rule_cu_file
   Kind: PythonRule
   Priority: Fallback
@@ -144,7 +157,7 @@
     value:
       MatchMode: Full
       In: ${arg}.cu
-      Out: ${arg}.cpp
+      Out: ${arg}.${rewrite_extention_name}
 
 - Rule: rule_cuda_device_count
   Kind: PythonRule