Ruby frontend (#94)

* Ruby frontend Signed-off-by: Prabhu Subramanian <[email protected]> Install ruby 3.4.0 Signed-off-by: Prabhu Subramanian <[email protected]> * Install ruby 3.4.0 Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Jan 4, 2025 · 8c42f2a · 8c42f2a
1 parent 3a859cf
commit 8c42f2a
Show file tree

Hide file tree

Showing 93 changed files with 8,663 additions and 192 deletions.
diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
@@ -48,6 +48,7 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: '22.x'
+      - uses: oras-project/setup-oras@v1
       - name: Trim CI agent
         run: |
           chmod +x ci/free_disk_space.sh

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -34,6 +34,7 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: '22.x'
+      - uses: oras-project/setup-oras@v1
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -40,6 +40,7 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: '22.x'
+      - uses: oras-project/setup-oras@v1
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'

diff --git a/build.sbt b/build.sbt
@@ -1,7 +1,7 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "2.2.3"
-ThisBuild / scalaVersion := "3.5.2"
+ThisBuild / version      := "2.3.0"
+ThisBuild / scalaVersion := "3.6.2"
 
 val cpgVersion = "1.0.1"
 
@@ -17,6 +17,7 @@ lazy val jssrc2cpg         = Projects.jssrc2cpg
 lazy val javasrc2cpg       = Projects.javasrc2cpg
 lazy val jimple2cpg        = Projects.jimple2cpg
 lazy val php2atom          = Projects.php2atom
+lazy val ruby2atom         = Projects.ruby2atom
 
 lazy val aggregatedProjects: Seq[ProjectReference] = Seq(
   platform,
@@ -30,7 +31,8 @@ lazy val aggregatedProjects: Seq[ProjectReference] = Seq(
   jssrc2cpg,
   javasrc2cpg,
   jimple2cpg,
-  php2atom
+  php2atom,
+  ruby2atom,
 )
 
 ThisBuild / libraryDependencies ++= Seq(

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -1,10 +1,10 @@
-FROM almalinux:9.4-minimal
+FROM ghcr.io/appthreat/base:main
 
 LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/chen" \
       org.opencontainers.image.url="https://github.com/appthreat/chen" \
-      org.opencontainers.image.version="2.2.x" \
+      org.opencontainers.image.version="2.3.x" \
       org.opencontainers.image.vendor="appthreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="chen" \
@@ -14,26 +14,33 @@ LABEL maintainer="appthreat" \
 ARG JAVA_VERSION=23.0.1-tem
 ARG MAVEN_VERSION=3.9.9
 ARG GRADLE_VERSION=8.11
+ARG RUBY_VERSION=3.4.0
 
 ENV JAVA_VERSION=$JAVA_VERSION \
     MAVEN_VERSION=$MAVEN_VERSION \
     GRADLE_VERSION=$GRADLE_VERSION \
+    RUBY_VERSION=$RUBY_VERSION \
     GRADLE_OPTS="-Dorg.gradle.daemon=false" \
     JAVA_HOME="/opt/java/${JAVA_VERSION}" \
     MAVEN_HOME="/opt/maven/${MAVEN_VERSION}" \
     GRADLE_HOME="/opt/gradle/${GRADLE_VERSION}" \
     PYTHON_CMD=python3 \
     PYTHONUNBUFFERED=1 \
     PYTHONIOENCODING="utf-8" \
-    JAVA_OPTS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1" \
+    JAVA_OPTS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1 -Dorg.jline.terminal.disableDeprecatedProviderWarning=true" \
     CHEN_DATAFLOW_TRACKED_WIDTH=128 \
     SCALAPY_PYTHON_LIBRARY=python3.12 \
     ANDROID_HOME=/opt/android-sdk-linux \
     CHEN_INSTALL_DIR=/opt/workspace \
     PHP_PARSER_BIN=/opt/vendor/bin/php-parse \
     CDXGEN_NO_BANNER=true \
-    COMPOSER_ALLOW_SUPERUSER=1
-ENV PATH=/opt/miniconda3/bin:${PATH}:/opt/platform:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${GRADLE_HOME}/bin:/usr/local/bin/:/root/.local/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:
+    COMPOSER_ALLOW_SUPERUSER=1 \
+    MALLOC_CONF="dirty_decay_ms:2000,narenas:2,background_thread:true" \
+    RUBY_CONFIGURE_OPTS="--with-jemalloc --enable-yjit" \
+    RUBYOPT="--yjit" \
+    RUBY_BUILD_BUILD_PATH="/tmp/rbenv" \
+    RUBY_BUILD_HTTP_CLIENT=curl
+ENV PATH=/opt/miniconda3/bin:${PATH}:/opt/platform:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${GRADLE_HOME}/bin:/usr/local/bin/:/root/.local/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:/root/.rbenv/bin:
 
 WORKDIR /opt
 COPY ./ci/conda-install.sh /opt/
@@ -57,7 +64,21 @@ RUN set -e; \
     esac; \
     echo -e "[nodejs]\nname=nodejs\nstream=20\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
     && microdnf install -y gcc git-core php php-cli php-curl php-zip php-bcmath php-json php-pear php-mbstring php-devel make wget bash graphviz graphviz-gd \
-        pcre2 findutils which tar gzip zip unzip sudo nodejs ncurses sqlite-devel glibc-common glibc-all-langpacks \
+        openssl-devel libffi-devel readline-devel libyaml zlib-devel ncurses ncurses-devel rust \
+        pcre2 findutils which tar gzip zip unzip sudo nodejs sqlite-devel glibc-common glibc-all-langpacks \
+    && microdnf install -y epel-release \
+    && microdnf install --enablerepo=crb -y libyaml-devel jemalloc-devel \
+    && git clone https://github.com/rbenv/rbenv.git --depth=1 ~/.rbenv \
+    && echo 'export PATH="/root/.rbenv/bin:$PATH"' >> ~/.bashrc \
+    && echo 'eval "$(/root/.rbenv/bin/rbenv init - bash)"' >> ~/.bashrc \
+    && source ~/.bashrc \
+    && mkdir -p "$(rbenv root)/plugins" \
+    && git clone https://github.com/rbenv/ruby-build.git --depth=1 "$(rbenv root)/plugins/ruby-build" \
+    && MAKE_OPTS=-j2 rbenv install ${RUBY_VERSION} \
+    && rbenv global ${RUBY_VERSION} \
+    && ruby --version \
+    && which ruby \
+    && rm -rf /root/.rbenv/cache $RUBY_BUILD_BUILD_PATH \
     && mkdir -p /opt/miniconda3 /opt/workspace \
     && wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${ARCH_NAME}.sh -O /opt/miniconda3/miniconda.sh \
     && bash /opt/miniconda3/miniconda.sh -b -u -p /opt/miniconda3 \
@@ -74,15 +95,14 @@ RUN set -e; \
     && rpm -ivh graphviz-devel-2.44.0-26.el9.${ARCH_NAME}.rpm \
     && rm graphviz-devel-2.44.0-26.el9.${ARCH_NAME}.rpm \
     && curl -s "https://get.sdkman.io" | bash \
-    && source "$HOME/.sdkman/bin/sdkman-init.sh" \
-    && echo -e "sdkman_auto_answer=true\nsdkman_selfupdate_feature=false\nsdkman_auto_env=true\nsdkman_curl_connect_timeout=20\nsdkman_curl_max_time=0" >> $HOME/.sdkman/etc/config \
+    && source "/root/.sdkman/bin/sdkman-init.sh" \
+    && echo -e "sdkman_auto_answer=true\nsdkman_selfupdate_feature=false\nsdkman_auto_env=true\nsdkman_curl_connect_timeout=20\nsdkman_curl_max_time=0" >> /root/.sdkman/etc/config \
     && sdk install java $JAVA_VERSION \
     && sdk install maven $MAVEN_VERSION \
     && sdk install gradle $GRADLE_VERSION \
     && sdk offline enable \
     && mv /root/.sdkman/candidates/* /opt/ \
     && rm -rf /root/.sdkman \
-    && microdnf install -y epel-release \
     && mkdir -p ${ANDROID_HOME}/cmdline-tools \
     && curl -L https://dl.google.com/android/repository/commandlinetools-linux-11076708_latest.zip -o ${ANDROID_HOME}/cmdline-tools/android_tools.zip \
     && unzip ${ANDROID_HOME}/cmdline-tools/android_tools.zip -d ${ANDROID_HOME}/cmdline-tools/ \
@@ -92,7 +112,7 @@ RUN set -e; \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platform-tools' --sdk_root=/opt/android-sdk-linux \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platforms;android-34' --sdk_root=/opt/android-sdk-linux \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'build-tools;34.0.0' --sdk_root=/opt/android-sdk-linux \
-    && sudo npm install -g @appthreat/atom @cyclonedx/cdxgen --omit=optional \
+    && npm install -g @appthreat/atom @cyclonedx/cdxgen --omit=optional \
     && php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php \
     && mv composer.phar /usr/local/bin/composer
 ENV LC_ALL=en_US.UTF-8 \

diff --git a/codemeta.json b/codemeta.json
@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/chen",
   "issueTracker": "https://github.com/AppThreat/chen/issues",
   "name": "chen",
-  "version": "2.2.3",
+  "version": "2.3.0",
   "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.",
   "applicationCategory": "code-analysis",
   "keywords": [

diff --git a/console/build.sbt b/console/build.sbt
@@ -3,7 +3,7 @@ name := "console"
 enablePlugins(JavaAppPackaging)
 
 val ScoptVersion          = "4.1.0"
-val CaskVersion           = "0.10.1"
+val CaskVersion           = "0.10.2"
 val CirceVersion          = "0.14.10"
 val ZeroturnaroundVersion = "1.17"
 
@@ -28,9 +28,9 @@ libraryDependencies ++= Seq(
   "com.lihaoyi"          %% "pprint"            % "0.9.0",
   "com.lihaoyi"          %% "cask"              % CaskVersion,
   "dev.scalapy"          %% "scalapy-core"      % "0.5.3",
-  "org.scala-lang.modules" % "scala-asm"        % "9.7.0-scala-2",
+  "org.scala-lang.modules" % "scala-asm"        % "9.7.1-scala-1",
   "org.scalatest"        %% "scalatest"         % Versions.scalatest % Test,
-  "org.scala-lang"       %% "scala3-compiler"   % "3.5.2"
+  "org.scala-lang"       %% "scala3-compiler"   % "3.6.2"
 )
 
 

diff --git a/dataflowengineoss/build.sbt b/dataflowengineoss/build.sbt
@@ -8,7 +8,7 @@ libraryDependencies ++= Seq(
   "io.circe"               %% "circe-generic"              % Versions.circe,
   "io.circe"               %% "circe-parser"               % Versions.circe,
   "org.scalatest"          %% "scalatest"                  % Versions.scalatest % Test,
-  "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4"
+  "org.scala-lang.modules" %% "scala-parallel-collections" % "1.1.0"
 )
 
 enablePlugins(Antlr4Plugin)

diff --git a/dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/semanticsloader/Parser.scala b/dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/semanticsloader/Parser.scala
@@ -22,7 +22,7 @@ object Semantics:
 
   def empty: Semantics = fromList(List())
 
-class Semantics private (methodToSemantic: mutable.Map[String, FlowSemantic]):
+class Semantics(methodToSemantic: mutable.Map[String, FlowSemantic]):
 
   /** The map below keeps a mapping between results of a regex and the regex string it matches. e.g.
     *

diff --git a/meta.yaml b/meta.yaml
@@ -1,4 +1,4 @@
-{% set version = "2.2.3" %}
+{% set version = "2.3.0" %}
 
 package:
   name: chen

diff --git a/platform/frontends/c2cpg/build.sbt b/platform/frontends/c2cpg/build.sbt
@@ -3,14 +3,14 @@ name := "c2cpg"
 dependsOn(Projects.semanticcpg, Projects.dataflowengineoss % Test, Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
-  "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4",
-  "org.eclipse.platform"    % "org.eclipse.equinox.common"       % "3.19.100",
-  "org.eclipse.platform"    % "org.eclipse.core.resources"       % "3.21.0" excludeAll(
+  "org.scala-lang.modules" %% "scala-parallel-collections" % "1.1.0",
+  "org.eclipse.platform"    % "org.eclipse.equinox.common"       % "3.19.200",
+  "org.eclipse.platform"    % "org.eclipse.core.resources"       % "3.22.0" excludeAll(
     ExclusionRule(organization = "com.ibm.icu", name = "icu4j"),
     ExclusionRule(organization = "org.eclipse.platform", name = "org.eclipse.jface"),
     ExclusionRule(organization = "org.eclipse.platform", name = "org.eclipse.jface.text")
   ),
-  "org.jline"               % "jline"                      % "3.27.1",
+  "org.jline"               % "jline"                      % "3.28.0",
   "org.scalatest"          %% "scalatest"                  % Versions.scalatest % Test
 )
 

diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/AstCreationPass.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/AstCreationPass.scala
@@ -31,7 +31,9 @@ class AstCreationPass(cpg: Cpg, config: Config, report: Report = new Report())
           .determine(
             config.inputPath,
             FileDefaults.SOURCE_FILE_EXTENSIONS ++ FileDefaults.HEADER_FILE_EXTENSIONS,
-            config.withDefaultIgnoredFilesRegex(DefaultIgnoredFolders)
+            ignoredDefaultRegex = Option(DefaultIgnoredFolders),
+            ignoredFilesRegex = Option(config.ignoredFilesRegex),
+            ignoredFilesPath = Option(config.ignoredFiles)
           )
           .sortWith(_.compareToIgnoreCase(_) > 0)
           .toArray

diff --git a/platform/frontends/javasrc2cpg/build.sbt b/platform/frontends/javasrc2cpg/build.sbt
@@ -4,10 +4,10 @@ dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->t
 
 libraryDependencies ++= Seq(
   "io.appthreat"           %% "cpg2"             % Versions.cpg,
-  "com.github.javaparser"   % "javaparser-symbol-solver-core" % "3.26.2",
+  "com.github.javaparser"   % "javaparser-symbol-solver-core" % "3.26.3",
   "org.scalatest"          %% "scalatest"                     % Versions.scalatest % Test,
   "org.projectlombok"       % "lombok"                        % "1.18.36",
-  "org.scala-lang.modules" %% "scala-parallel-collections"    % "1.0.4",
+  "org.scala-lang.modules" %% "scala-parallel-collections"    % "1.1.0",
   "org.scala-lang.modules" %% "scala-parser-combinators"      % "2.4.0",
   "net.lingala.zip4j"       % "zip4j"                         % "2.11.5"
 )

diff --git a/...orm/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/util/SourceParser.scala b/...orm/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/util/SourceParser.scala
@@ -90,8 +90,13 @@ object SourceParser:
     config: Config,
     sourcesOverride: Option[List[String]] = None
   ): Array[String] =
-    val inputPaths = sourcesOverride.getOrElse(config.inputPath :: Nil).toSet
-    SourceFiles.determine(inputPaths, JavaSrc2Cpg.sourceFileExtensions, config).toArray
+      SourceFiles.determine(
+        config.inputPath,
+        JavaSrc2Cpg.sourceFileExtensions,
+        ignoredDefaultRegex = Option(JavaSrc2Cpg.DefaultIgnoredFilesRegex),
+        ignoredFilesRegex = Option(config.ignoredFilesRegex),
+        ignoredFilesPath = Option(config.ignoredFiles)
+      ).toArray
 
   /** Implements the logic described in the option description for the "delombok-mode" option:
     *   - no-delombok: do not run delombok.

diff --git a/platform/frontends/jimple2cpg/AUTHORS b/platform/frontends/jimple2cpg/AUTHORS
diff --git a/platform/frontends/jimple2cpg/build.sbt b/platform/frontends/jimple2cpg/build.sbt
@@ -4,13 +4,14 @@ dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->t
 
 libraryDependencies ++= Seq(
   "io.appthreat"           %% "cpg2"              % Versions.cpg,
-  "commons-io"             % "commons-io"         % "2.17.0",
+  "commons-io"             % "commons-io"         % "2.18.0",
   "org.soot-oss"           % "soot"               % "4.6.0",
-  "org.scala-lang.modules" % "scala-asm"          % "9.7.0-scala-2",
+  "org.scala-lang.modules" % "scala-asm"          % "9.7.1-scala-1",
   "org.ow2.asm"            % "asm"                % "9.7.1",
   "org.ow2.asm"            % "asm-analysis"       % "9.7.1",
   "org.ow2.asm"            % "asm-util"           % "9.7.1",
   "org.ow2.asm"            % "asm-tree"           % "9.7.1",
+  "io.circe"               %% "circe-core"        % Versions.circe,
   "org.scalatest" %% "scalatest"                  % Versions.scalatest % Test
 )
 

diff --git a/platform/frontends/jimple2cpg/src/test/resources/project/build.properties b/platform/frontends/jimple2cpg/src/test/resources/project/build.properties
@@ -1 +1 @@
-sbt.version=1.5.2
+sbt.version=1.10.7
diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/PhpParser.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/PhpParser.scala
@@ -4,15 +4,12 @@ import better.files.File
 import io.appthreat.php2atom.Config
 import io.appthreat.php2atom.parser.Domain.PhpFile
 import io.appthreat.x2cpg.utils.ExternalCommand
-import org.slf4j.LoggerFactory
 
 import java.nio.file.Paths
 import scala.util.{Failure, Success, Try}
 
 class PhpParser private (phpParserPath: String, phpIniPath: String):
 
-  private val logger = LoggerFactory.getLogger(this.getClass)
-
   private def phpParseCommand(filename: String): String =
     val phpParserCommands = "--with-recovery --resolve-names -P --json-dump"
     phpParserPath match
@@ -33,7 +30,6 @@ class PhpParser private (phpParserPath: String, phpIniPath: String):
           processParserOutput(output, inputFilePath)
 
       case Failure(exception) =>
-          logger.debug(s"Failure running php-parser with $command", exception.getMessage)
           None
 
   private def processParserOutput(output: Seq[String], filename: String): Option[PhpFile] =
@@ -48,30 +44,22 @@ class PhpParser private (phpParserPath: String, phpIniPath: String):
           case Success(Some(value)) => Some(value)
 
           case Success(None) =>
-              logger.debug(s"Parsing json string for $filename resulted in null return value")
               None
 
           case Failure(exception) =>
-              logger.debug(
-                s"Parsing json string for $filename failed with exception",
-                exception
-              )
               None
       else
-        logger.debug(s"No JSON output for $filename")
         None
 
   private def jsonValueToPhpFile(json: ujson.Value, filename: String): Option[PhpFile] =
       Try(Domain.fromJson(json)) match
         case Success(phpFile) => Some(phpFile)
 
         case Failure(e) =>
-            logger.debug(s"Failed to generate intermediate AST for $filename", e)
             None
 end PhpParser
 
 object PhpParser:
-  private val logger = LoggerFactory.getLogger(this.getClass())
 
   val PhpParserBinEnvVar = "PHP_PARSER_BIN"
 

diff --git a/platform/frontends/pysrc2cpg/build.sbt b/platform/frontends/pysrc2cpg/build.sbt
@@ -4,7 +4,7 @@ dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->t
 
 libraryDependencies ++= Seq(
   "io.appthreat"           %% "cpg2"          % Versions.cpg,
-  "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4",
+  "org.scala-lang.modules" %% "scala-parallel-collections" % "1.1.0",
   "org.scalatest"          %% "scalatest"                  % Versions.scalatest % Test
 )