From c2f9ada3688cb5d9a919140bb8f04980d999c6f1 Mon Sep 17 00:00:00 2001 From: prabhu Date: Fri, 22 Dec 2023 09:55:34 +0000 Subject: [PATCH] php2atom with php-parser (#47) Signed-off-by: Prabhu Subramanian --- .dockerignore | 3 +- .github/workflows/containers.yml | 6 + .github/workflows/master.yml | 7 + .github/workflows/pr.yml | 7 + .github/workflows/release.yml | 6 + .gitignore | 3 +- build.sbt | 4 +- ci/Dockerfile | 13 +- codemeta.json | 2 +- meta.yaml | 2 +- platform/frontends/c2cpg/README.md | 38 - platform/frontends/javasrc2cpg/README.md | 54 - .../passes/ConfigFileCreationPass.scala | 4 +- platform/frontends/jimple2cpg/README.md | 37 - platform/frontends/jssrc2cpg/README.md | 49 - platform/frontends/php2atom/build.sbt | 26 + platform/frontends/php2atom/composer.json | 15 + platform/frontends/php2atom/composer.lock | 75 + platform/frontends/php2atom/install.sh | 5 + .../src/main/resources/builtin_functions.txt | 2332 +++++++++++++++++ .../resources/known_function_signatures.txt | 56 + .../php2atom/src/main/resources/log4j2.xml | 13 + .../php2atom/src/main/resources/php.ini | 2 + .../scala/io/appthreat/php2atom/Main.scala | 39 + .../io/appthreat/php2atom/Php2Atom.scala | 69 + .../php2atom/astcreation/AstCreator.scala | 1860 +++++++++++++ .../php2atom/astcreation/PhpBuiltins.scala | 9 + .../io/appthreat/php2atom/parser/Domain.scala | 1442 ++++++++++ .../appthreat/php2atom/parser/PhpParser.scala | 148 ++ .../php2atom/passes/AnyTypePass.scala | 21 + .../php2atom/passes/AstCreationPass.scala | 43 + .../php2atom/passes/AstParentInfoPass.scala | 36 + .../php2atom/passes/ClosureRefPass.scala | 73 + .../php2atom/passes/LocalCreationPass.scala | 135 + .../php2atom/passes/PhpSetKnownTypes.scala | 79 + .../php2atom/utils/ArrayIndexTracker.scala | 16 + .../io/appthreat/php2atom/utils/Scope.scala | 106 + .../php2atom/utils/ScopeElement.scala | 31 + .../src/test/resources/builtin_functions.txt | 1 + .../php2atom/config/ConfigTests.scala | 41 + .../dataflow/IntraMethodDataflowTests.scala | 32 + .../passes/CfgCreationPassTests.scala | 186 ++ .../php2atom/querying/ArrayTests.scala | 187 ++ .../php2atom/querying/CallTests.scala | 170 ++ .../php2atom/querying/CfgTests.scala | 269 ++ .../php2atom/querying/ClosureTests.scala | 172 ++ .../php2atom/querying/CommentTests.scala | 17 + .../querying/ControlStructureTests.scala | 1228 +++++++++ .../php2atom/querying/FieldAccessTests.scala | 94 + .../php2atom/querying/LocalTests.scala | 81 + .../php2atom/querying/MemberTests.scala | 223 ++ .../php2atom/querying/MethodTests.scala | 155 ++ .../php2atom/querying/NamespaceTests.scala | 163 ++ .../php2atom/querying/OperatorTests.scala | 670 +++++ .../appthreat/php2atom/querying/PocTest.scala | 75 + .../php2atom/querying/ScalarTests.scala | 87 + .../php2atom/querying/TypeDeclTests.scala | 286 ++ .../php2atom/querying/TypeNodeTests.scala | 38 + .../php2atom/querying/UseTests.scala | 75 + .../testfixtures/PhpCode2CpgFixture.scala | 44 + platform/frontends/pysrc2cpg/README.md | 9 - .../io/appthreat/x2cpg/AstCreatorBase.scala | 5 +- .../scala/io/appthreat/x2cpg/Defines.scala | 1 + project/Projects.scala | 1 + pyproject.toml | 2 +- 65 files changed, 10980 insertions(+), 198 deletions(-) delete mode 100644 platform/frontends/c2cpg/README.md delete mode 100644 platform/frontends/javasrc2cpg/README.md delete mode 100644 platform/frontends/jimple2cpg/README.md delete mode 100644 platform/frontends/jssrc2cpg/README.md create mode 100644 platform/frontends/php2atom/build.sbt create mode 100644 platform/frontends/php2atom/composer.json create mode 100644 platform/frontends/php2atom/composer.lock create mode 100644 platform/frontends/php2atom/install.sh create mode 100644 platform/frontends/php2atom/src/main/resources/builtin_functions.txt create mode 100644 platform/frontends/php2atom/src/main/resources/known_function_signatures.txt create mode 100644 platform/frontends/php2atom/src/main/resources/log4j2.xml create mode 100644 platform/frontends/php2atom/src/main/resources/php.ini create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/Main.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/Php2Atom.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/astcreation/AstCreator.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/astcreation/PhpBuiltins.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/Domain.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/PhpParser.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AnyTypePass.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AstCreationPass.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AstParentInfoPass.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/ClosureRefPass.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/LocalCreationPass.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/PhpSetKnownTypes.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/ArrayIndexTracker.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/Scope.scala create mode 100644 platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/ScopeElement.scala create mode 100644 platform/frontends/php2atom/src/test/resources/builtin_functions.txt create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/config/ConfigTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/dataflow/IntraMethodDataflowTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/passes/CfgCreationPassTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/ArrayTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CallTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CfgTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/ClosureTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CommentTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/ControlStructureTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/FieldAccessTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/LocalTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/MemberTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/MethodTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/NamespaceTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/OperatorTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/PocTest.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/ScalarTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/TypeDeclTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/TypeNodeTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/UseTests.scala create mode 100644 platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/testfixtures/PhpCode2CpgFixture.scala delete mode 100644 platform/frontends/pysrc2cpg/README.md diff --git a/.dockerignore b/.dockerignore index ee1beac7..09f46ffc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,7 +21,8 @@ platform/workspace **/astgen-macos **/astgen-macos-arm **/astgen-linux -**/php2cpg/bin +**/php2atom/bin +**/php2atom/vendor /foo.c /woo.c .DS_Store diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index fd630042..4972ccbf 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -28,6 +28,12 @@ jobs: with: distribution: 'zulu' java-version: '21' + - name: "Install PHP" + uses: "shivammathur/setup-php@v2" + with: + coverage: "none" + php-version: "8.3" + tools: composer:v2 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 477a4158..1e139bc7 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,6 +16,12 @@ jobs: with: distribution: 'zulu' java-version: '21' + - name: "Install PHP" + uses: "shivammathur/setup-php@v2" + with: + coverage: "none" + php-version: "8.3" + tools: composer:v2 - name: Set up Python uses: actions/setup-python@v4 with: @@ -44,6 +50,7 @@ jobs: if: runner.os == 'macOS' - name: Install and test run: | + bash ./platform/frontends/php2atom/install.sh npm install -g @appthreat/atom python3.11 -m pip install --upgrade pip python3.11 -m pip install poetry diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 194f638c..ec354800 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,6 +16,12 @@ jobs: with: distribution: 'zulu' java-version: ${{ matrix.jvm }} + - name: "Install PHP" + uses: "shivammathur/setup-php@v2" + with: + coverage: "none" + php-version: "8.3" + tools: composer:v2 - name: Set up Python uses: actions/setup-python@v4 with: @@ -44,6 +50,7 @@ jobs: if: runner.os == 'macOS' - name: Install and test run: | + bash ./platform/frontends/php2atom/install.sh npm install -g @appthreat/atom python3 -m pip install --upgrade pip setuptools wheel python3 -m pip install poetry diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c60e16d9..4f13586f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,12 @@ jobs: with: distribution: 'zulu' java-version: '21' + - name: "Install PHP" + uses: "shivammathur/setup-php@v2" + with: + coverage: "none" + php-version: "8.3" + tools: composer:v2 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.gitignore b/.gitignore index 26135b3e..a2005d85 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,8 @@ null **/astgen-macos **/astgen-macos-arm **/astgen-linux -**/php2cpg/bin +**/php2atom/bin +**/php2atom/vendor /foo.c /woo.c /cpg_*.bin.zip diff --git a/build.sbt b/build.sbt index 654e7914..e5a0ff3f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "chen" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "1.1.2" +ThisBuild / version := "1.1.3" ThisBuild / scalaVersion := "3.3.1" val cpgVersion = "1.4.22" @@ -16,6 +16,7 @@ lazy val pysrc2cpg = Projects.pysrc2cpg lazy val jssrc2cpg = Projects.jssrc2cpg lazy val javasrc2cpg = Projects.javasrc2cpg lazy val jimple2cpg = Projects.jimple2cpg +lazy val php2atom = Projects.php2atom lazy val aggregatedProjects: Seq[ProjectReference] = Seq( platform, @@ -29,6 +30,7 @@ lazy val aggregatedProjects: Seq[ProjectReference] = Seq( jssrc2cpg, javasrc2cpg, jimple2cpg, + php2atom ) ThisBuild / libraryDependencies ++= Seq( diff --git a/ci/Dockerfile b/ci/Dockerfile index 4811d22a..864cb46e 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -29,12 +29,16 @@ ENV JAVA_VERSION=$JAVA_VERSION \ CHEN_DATAFLOW_TRACKED_WIDTH=128 \ SCALAPY_PYTHON_LIBRARY=python3.11 \ ANDROID_HOME=/opt/android-sdk-linux \ - CHEN_INSTALL_DIR=/opt/workspace + CHEN_INSTALL_DIR=/opt/workspace \ + PHP_PARSER_BIN=/opt/vendor/bin/php-parse \ + COMPOSER_ALLOW_SUPERUSER=1 ENV PATH=/opt/miniconda3/bin:${PATH}:/opt/platform:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${GRADLE_HOME}/bin:/usr/local/bin/:/root/.local/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools: WORKDIR /opt COPY ./ci/conda-install.sh /opt/ COPY README.md /opt/ +COPY ./platform/frontends/php2atom/composer.json /opt/composer.json +COPY ./platform/frontends/php2atom/composer.json /opt/composer.lock RUN set -e; \ ARCH_NAME="$(rpm --eval '%{_arch}')"; \ @@ -51,7 +55,7 @@ RUN set -e; \ *) echo >&2 "error: unsupported architecture: '$ARCH_NAME'"; exit 1 ;; \ esac; \ echo -e "[nodejs]\nname=nodejs\nstream=20\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \ - && microdnf install -y gcc git-core wget bash graphviz graphviz-gd \ + && microdnf install -y gcc git-core php php-cli php-curl php-zip php-bcmath php-json php-pear php-mbstring php-devel make wget bash graphviz graphviz-gd \ pcre2 findutils which tar gzip zip unzip sudo nodejs ncurses sqlite-devel glibc-common glibc-all-langpacks \ && mkdir -p /opt/miniconda3 /opt/workspace \ && wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /opt/miniconda3/miniconda.sh \ @@ -87,7 +91,9 @@ RUN set -e; \ && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platform-tools' --sdk_root=/opt/android-sdk-linux \ && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platforms;android-33' --sdk_root=/opt/android-sdk-linux \ && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'build-tools;33.0.0' --sdk_root=/opt/android-sdk-linux \ - && sudo npm install -g @appthreat/atom @cyclonedx/cdxgen --omit=optional + && sudo npm install -g @appthreat/atom @cyclonedx/cdxgen --omit=optional \ + && php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php \ + && mv composer.phar /usr/local/bin/composer ENV LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 \ LANGUAGE=en_US.UTF-8 \ @@ -97,6 +103,7 @@ COPY ./pyproject.toml /opt/ COPY ./target/chen.zip . COPY ./notebooks /opt/notebooks RUN unzip -q chen.zip \ + && composer update --no-progress --prefer-dist --ignore-platform-reqs \ && python -m pip install --no-deps . \ && rm chen.zip conda-install.sh pyproject.toml \ && microdnf clean all diff --git a/codemeta.json b/codemeta.json index 2b883b13..dd451527 100644 --- a/codemeta.json +++ b/codemeta.json @@ -7,7 +7,7 @@ "downloadUrl": "https://github.com/AppThreat/chen", "issueTracker": "https://github.com/AppThreat/chen/issues", "name": "chen", - "version": "1.1.2", + "version": "1.1.3", "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.", "applicationCategory": "code-analysis", "keywords": [ diff --git a/meta.yaml b/meta.yaml index 0b78cc0e..ce07f004 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,4 +1,4 @@ -{% set version = "1.1.2" %} +{% set version = "1.1.3" %} package: name: chen diff --git a/platform/frontends/c2cpg/README.md b/platform/frontends/c2cpg/README.md deleted file mode 100644 index bd4cb374..00000000 --- a/platform/frontends/c2cpg/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# c2cpg - -An [Eclipse CDT](https://wiki.eclipse.org/CDT/designs/Overview_of_Parsing) based parser for C/C++ that creates code property graphs according to the specification at https://github.com/ShiftLeftSecurity/codepropertygraph . - -## Building the code - -The build process has been verified on Linux, and it should be possible -to build on OS X and BSD systems as well. The build process requires -the following prerequisites: - -* Java runtime 17 - - Link: http://openjdk.java.net/install/ -* Scala build tool (sbt) - - Link: https://www.scala-sbt.org/ - -Additional build-time dependencies are automatically downloaded as part -of the build process. To build c2cpg issue the command `sbt stage`. - -## Running - -To produce a code property graph issue the command: -```shell script -./c2cpg.sh --output -````` - -Additional options are available: -```shell script -./c2cpg.sh \ - --output \ - --include , - --define DEF - --define DEF_VAL=2 -``` - -Run the following to see a complete list of available options: -```shell script -./c2cpg.sh --help -``` diff --git a/platform/frontends/javasrc2cpg/README.md b/platform/frontends/javasrc2cpg/README.md deleted file mode 100644 index 0fefc1a3..00000000 --- a/platform/frontends/javasrc2cpg/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# JavaSrc2cpg - -This is a [CPG](https://docs.joern.io/code-property-graph/) frontend based on Java source code powered by -[JavaParser](https://javaparser.org). - -[![Discord](https://img.shields.io/badge/-Discord-lime?style=for-the-badge&logo=discord&logoColor=white&color=black)](https://discord.com/invite/vv4MH284Hc) - -## Setup - -Requirements: -- \>= JDK 11. We recommend OpenJDK 11. -- sbt (https://www.scala-sbt.org/) - -### Quickstart - - -1. From the `joern` root directory, run `sbt stage` -2. Start Joern with `./joern.sh` -3. Import your code with `importCode.javasrc("") -4. Now you can query the CPG - -### Development - -Some general development habits for the project: - -- When making a branch, use the following template `/` - e.g. `fabs/control-structure-nodes`. -- We currently focus around test driven development. Pay attention to the code coverage when creating new tests and - features. The code coverage report can be found under `./target/scala-2.13/scoverage-report`. - -### TODO -- [x] Explicit constructor invocations -- [x] Propagate context up and down while creating AST -- [x] Handle body of `try`/`catch` -- [ ] `code` field for method calls (`this` for `System.out` in `System.out.println`) -- [ ] Logging -- [ ] Local class declaration statements -- [ ] Lambda expressions (dataflow still to-do) -- [ ] Method Reference Expr -- [ ] Throw statements (AST is simple, control flow to catch seems harder) -- [ ] `this`/`super` expressions (scope for FieldAccess) -- [ ] Type expressions (part of MethodReferenceExpr) -- [ ] `instanceof` -- [ ] Pattern expr as part of `instanceof` (Java 14) -- [ ] `switch` expressions (including `yield` statements) (Introduced Java 12/13) -- [ ] Local record declaration statements (Java 14 Preview feature) -- [ ] Dataflow tests for inheritance - -### MAYBE TODO -- [ ] Type arguments for generics -- [ ] Annotations -- [ ] Cast expressions (maybe not necessary if `javaparser` resolves types correctly) -- [ ] Synchronized statements (if we don't just ignore those) -- [ ] Control flow for labeled breaks diff --git a/platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/passes/ConfigFileCreationPass.scala b/platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/passes/ConfigFileCreationPass.scala index 50898871..d66668d8 100644 --- a/platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/passes/ConfigFileCreationPass.scala +++ b/platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/passes/ConfigFileCreationPass.scala @@ -39,7 +39,9 @@ class ConfigFileCreationPass(cpg: Cpg) extends XConfigFileCreationPass(cpg): // Bom pathEndFilter("bom.json"), pathEndFilter(".cdx.json"), - pathEndFilter("chennai.json") + pathEndFilter("chennai.json"), + extensionFilter(".yml"), + extensionFilter(".yaml") ) private def mybatisFilter(file: File): Boolean = diff --git a/platform/frontends/jimple2cpg/README.md b/platform/frontends/jimple2cpg/README.md deleted file mode 100644 index 9f116872..00000000 --- a/platform/frontends/jimple2cpg/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# jimple2cpg - -This is a [CPG](https://docs.joern.io/code-property-graph/) frontend for Soot's -Jimple IR. This language frontend is formerly known as -[Plume](https://plume-oss.github.io/plume-docs/). - - -## Setup - -Requirements: -- \>= JDK 11. We recommend OpenJDK 11. -- sbt (https://www.scala-sbt.org/) - -### Quickstart - -1. Clone the project -2. Build the project `sbt stage` -3. Create a CPG `./jimple2cpg.sh /path/to/your/code -o /path/to/cpg.bin` -4. Download Joern with - ``` - wget https://github.com/appthreat/joern/releases/latest/download/platform.zip - unzip platform.zip - cd platform - ``` -5. Copy `cpg.bin` into the Joern directory -6. Start Joern with `./joern.sh` -7. Import the cpg with `importCpg("cpg.bin")` -8. Now you can query the CPG - -### Development - -Some general development habits for the project: - -- When making a branch, use the following template `/` - e.g. `fabs/control-structure-nodes`. -- We currently focus around test driven development. Pay attention to the code coverage when creating new tests and - features. The code coverage report can be found under `./target/scala-2.13/scoverage-report`. \ No newline at end of file diff --git a/platform/frontends/jssrc2cpg/README.md b/platform/frontends/jssrc2cpg/README.md deleted file mode 100644 index 541a81b3..00000000 --- a/platform/frontends/jssrc2cpg/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# jssrc2cpg - -A Babel based parser for JavaScript/TypeScript that creates code property graphs according to the specification at https://github.com/ShiftLeftSecurity/codepropertygraph . - -## Building the code - -The build process has been verified on Linux, and it should be possible -to build on OS X and BSD systems as well. The build process requires -the following prerequisites: - -* Java runtime 11 - - Link: http://openjdk.java.net/install/ -* Scala build tool (sbt) - - Link: https://www.scala-sbt.org/ - -Additional build-time dependencies are automatically downloaded as part -of the build process. To build jssrc2cpg issue the command `sbt stage`. - -## JS/TS AST Generation - -jssrc2cpg uses [@joern/astgen](https://github.com/joernio/astgen) under the hood. -Native binaries for Linux, macOS, and Windows are generated as described [here](https://github.com/joernio/astgen#building). -To build your own native binaries run the following commands: - -```shell script -git clone https://github.com/joernio/astgen.git -cd astgen -yarn install -``` -(requires `yarn`). - -Copy the resulting `astgen-linux`, `astgen-macos`, `astgen-macos-arm`, and `astgen-win.exe` to `joern/platform/frontends/jssrc2cpg/bin/astgen`. - -## Running - -To produce a code property graph issue the command: -```shell script -./jssrc2cpg.sh --output -````` - -Run the following to see a complete list of available options: -```shell script -./jssrc2cpg.sh --help -``` - -## Warning - -This is work in progress. Use https://github.com/ShiftLeftSecurity/js2cpg as a mature alternative. - diff --git a/platform/frontends/php2atom/build.sbt b/platform/frontends/php2atom/build.sbt new file mode 100644 index 00000000..827eb40d --- /dev/null +++ b/platform/frontends/php2atom/build.sbt @@ -0,0 +1,26 @@ +name := "php2atom" + +dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test") + +libraryDependencies ++= Seq( + "com.lihaoyi" %% "upickle" % Versions.upickle, + "com.lihaoyi" %% "ujson" % Versions.upickle, + "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "org.scalatest" %% "scalatest" % Versions.scalatest % Test, + "io.circe" %% "circe-core" % Versions.circe +) + +enablePlugins(JavaAppPackaging, LauncherJarPlugin) +Global / onChangedBuildSource := ReloadOnSourceChanges +Universal / packageName := name.value +Universal / topLevelDirectory := None + +githubOwner := "appthreat" +githubRepository := "chen" +credentials += + Credentials( + "GitHub Package Registry", + "maven.pkg.github.com", + "appthreat", + sys.env.getOrElse("GITHUB_TOKEN", "N/A") + ) diff --git a/platform/frontends/php2atom/composer.json b/platform/frontends/php2atom/composer.json new file mode 100644 index 00000000..1e61654b --- /dev/null +++ b/platform/frontends/php2atom/composer.json @@ -0,0 +1,15 @@ +{ + "name": "appthreat/php2atom", + "description": "Convert PHP source to atom", + "type": "library", + "require": { + "nikic/php-parser": "^4.18" + }, + "license": "Apache-2.0", + "authors": [ + { + "name": "Team AppThreat", + "email": "cloud@appthreat.com" + } + ] +} diff --git a/platform/frontends/php2atom/composer.lock b/platform/frontends/php2atom/composer.lock new file mode 100644 index 00000000..a550da48 --- /dev/null +++ b/platform/frontends/php2atom/composer.lock @@ -0,0 +1,75 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically" + ], + "content-hash": "528202ec59a6f02d0ec1a2280f503842", + "packages": [ + { + "name": "nikic/php-parser", + "version": "v4.18.0", + "source": { + "type": "git", + "url": "https://github.com/nikic/PHP-Parser.git", + "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/1bcbb2179f97633e98bbbc87044ee2611c7d7999", + "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999", + "shasum": "" + }, + "require": { + "ext-tokenizer": "*", + "php": ">=7.0" + }, + "require-dev": { + "ircmaxell/php-yacc": "^0.0.7", + "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0" + }, + "bin": [ + "bin/php-parse" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.9-dev" + } + }, + "autoload": { + "psr-4": { + "PhpParser\\": "lib/PhpParser" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Nikita Popov" + } + ], + "description": "A PHP parser written in PHP", + "keywords": [ + "parser", + "php" + ], + "support": { + "issues": "https://github.com/nikic/PHP-Parser/issues", + "source": "https://github.com/nikic/PHP-Parser/tree/v4.18.0" + }, + "time": "2023-12-10T21:03:43+00:00" + } + ], + "packages-dev": [], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": [], + "prefer-stable": false, + "prefer-lowest": false, + "platform": [], + "platform-dev": [], + "plugin-api-version": "2.6.0" +} diff --git a/platform/frontends/php2atom/install.sh b/platform/frontends/php2atom/install.sh new file mode 100644 index 00000000..52a9f475 --- /dev/null +++ b/platform/frontends/php2atom/install.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +pushd $(dirname $0) +composer update --no-progress --prefer-dist --ignore-platform-reqs +popd +export PHP_PARSER_BIN="$(dirname $0)/vendor/bin/php-parse" diff --git a/platform/frontends/php2atom/src/main/resources/builtin_functions.txt b/platform/frontends/php2atom/src/main/resources/builtin_functions.txt new file mode 100644 index 00000000..97b8c167 --- /dev/null +++ b/platform/frontends/php2atom/src/main/resources/builtin_functions.txt @@ -0,0 +1,2332 @@ +abs +acos +acosh +addcslashes +addslashes +apache_child_terminate +apache_get_modules +apache_get_version +apache_getenv +apache_lookup_uri +apache_note +apache_request_headers +apache_response_headers +apache_setenv +apcu_add +apcu_cache_info +apcu_cas +apcu_clear_cache +apcu_dec +apcu_delete +apcu_enabled +apcu_entry +apcu_exists +apcu_fetch +apcu_inc +apcu_key_info +apcu_sma_info +apcu_store +array +array_change_key_case +array_chunk +array_column +array_combine +array_count_values +array_diff +array_diff_assoc +array_diff_key +array_diff_uassoc +array_diff_ukey +array_fill +array_fill_keys +array_filter +array_flip +array_intersect +array_intersect_assoc +array_intersect_key +array_intersect_uassoc +array_intersect_ukey +array_is_list +array_key_exists +array_key_first +array_key_last +array_keys +array_map +array_merge +array_merge_recursive +array_multisort +array_pad +array_pop +array_product +array_push +array_rand +array_reduce +array_replace +array_replace_recursive +array_reverse +array_search +array_shift +array_slice +array_splice +array_sum +array_udiff +array_udiff_assoc +array_udiff_uassoc +array_uintersect +array_uintersect_assoc +array_uintersect_uassoc +array_unique +array_unshift +array_values +array_walk +array_walk_recursive +arsort +asin +asinh +asort +assert +assert_options +atan +atan2 +atanh +base64_decode +base64_encode +base_convert +basename +bcadd +bccomp +bcdiv +bcmod +bcmul +bcpow +bcpowmod +bcscale +bcsqrt +bcsub +bin2hex +bind_textdomain_codeset +bindec +bindtextdomain +boolval +bzclose +bzcompress +bzdecompress +bzerrno +bzerror +bzerrstr +bzflush +bzopen +bzread +bzwrite +cal_days_in_month +cal_from_jd +cal_info +cal_to_jd +call_user_func +call_user_func_array +ceil +chdir +checkdate +checkdnsrr +chgrp +chmod +chown +chr +chroot +chunk_split +class_alias +class_exists +class_implements +class_parents +class_uses +clearstatcache +cli_get_process_title +cli_set_process_title +closedir +closelog +com_create_guid +com_event_sink +com_get_active_object +com_load_typelib +com_message_pump +com_print_typeinfo +compact +connection_aborted +connection_status +constant +convert_cyr_string +convert_uudecode +convert_uuencode +copy +cos +cosh +count +count_chars +crc32 +create_function +crypt +ctype_alnum +ctype_alpha +ctype_cntrl +ctype_digit +ctype_graph +ctype_lower +ctype_print +ctype_punct +ctype_space +ctype_upper +ctype_xdigit +cubrid_affected_rows +curl_close +curl_copy_handle +curl_errno +curl_error +curl_escape +curl_exec +curl_getinfo +curl_init +curl_multi_add_handle +curl_multi_close +curl_multi_errno +curl_multi_exec +curl_multi_getcontent +curl_multi_info_read +curl_multi_init +curl_multi_remove_handle +curl_multi_select +curl_multi_setopt +curl_multi_strerror +curl_pause +curl_reset +curl_setopt +curl_setopt_array +curl_share_close +curl_share_errno +curl_share_init +curl_share_setopt +curl_share_strerror +curl_strerror +curl_unescape +curl_upkeep +curl_version +current +date +date_create +date_default_timezone_get +date_default_timezone_set +date_parse +date_parse_from_format +date_sun_info +date_sunrise +date_sunset +dba_close +dba_delete +dba_exists +dba_fetch +dba_firstkey +dba_handlers +dba_insert +dba_key_split +dba_list +dba_nextkey +dba_open +dba_optimize +dba_popen +dba_replace +dba_sync +dbase_add_record +dbase_close +dbase_create +dbase_delete_record +dbase_get_header_info +dbase_get_record +dbase_get_record_with_names +dbase_numfields +dbase_numrecords +dbase_open +dbase_pack +dbase_replace_record +dcgettext +dcngettext +debug_backtrace +debug_print_backtrace +debug_zval_dump +decbin +dechex +decoct +define +defined +deflate_add +deflate_init +deg2rad +delete +dgettext +die +dio_close +dio_fcntl +dio_open +dio_read +dio_seek +dio_stat +dio_tcsetattr +dio_truncate +dio_write +dir +dirname +disk_free_space +disk_total_space +dl +dngettext +dns_get_record +dom_import_simplexml +each +easter_date +easter_days +echo +empty +enchant_broker_describe +enchant_broker_dict_exists +enchant_broker_free +enchant_broker_free_dict +enchant_broker_get_dict_path +enchant_broker_get_error +enchant_broker_init +enchant_broker_list_dicts +enchant_broker_request_dict +enchant_broker_request_pwl_dict +enchant_broker_set_dict_path +enchant_broker_set_ordering +enchant_dict_add +enchant_dict_add_to_session +enchant_dict_check +enchant_dict_describe +enchant_dict_get_error +enchant_dict_is_added +enchant_dict_quick_check +enchant_dict_store_replacement +enchant_dict_suggest +end +enum_exists +error_clear_last +error_get_last +error_log +error_reporting +escapeshellarg +escapeshellcmd +eval +exec +exif_imagetype +exif_read_data +exif_tagname +exif_thumbnail +exit +exp +expect_expectl +expect_popen +explode +expm1 +expression +extension_loaded +extract +ezmlm_hash +fastcgi_finish_request +fbird_blob_cancel +fclose +fdatasync +fdf_add_doc_javascript +fdf_add_template +fdf_close +fdf_create +fdf_enum_values +fdf_errno +fdf_error +fdf_get_ap +fdf_get_attachment +fdf_get_encoding +fdf_get_file +fdf_get_flags +fdf_get_opt +fdf_get_status +fdf_get_value +fdf_get_version +fdf_header +fdf_next_field_name +fdf_open +fdf_open_string +fdf_remove_item +fdf_save +fdf_save_string +fdf_set_ap +fdf_set_encoding +fdf_set_file +fdf_set_flags +fdf_set_javascript_action +fdf_set_on_import_javascript +fdf_set_opt +fdf_set_status +fdf_set_submit_form_action +fdf_set_target_frame +fdf_set_value +fdf_set_version +fdiv +feof +fflush +fgetc +fgetcsv +fgets +fgetss +file +file_exists +file_get_contents +file_put_contents +fileatime +filectime +filegroup +fileinode +filemtime +fileowner +fileperms +filesize +filetype +filter_has_var +filter_id +filter_input +filter_input_array +filter_list +filter_var +filter_var_array +finfo_close +finfo_open +floatval +flock +floor +flush +fmod +fnmatch +fopen +forward_static_call +forward_static_call_array +fpassthru +fpm_get_status +fprintf +fputcsv +fread +frenchtojd +fscanf +fseek +fsockopen +fstat +fsync +ftell +ftok +ftp_alloc +ftp_append +ftp_cdup +ftp_chdir +ftp_chmod +ftp_close +ftp_connect +ftp_delete +ftp_exec +ftp_fget +ftp_fput +ftp_get +ftp_get_option +ftp_login +ftp_mdtm +ftp_mkdir +ftp_mlsd +ftp_nb_continue +ftp_nb_fget +ftp_nb_fput +ftp_nb_get +ftp_nb_put +ftp_nlist +ftp_pasv +ftp_put +ftp_pwd +ftp_raw +ftp_rawlist +ftp_rename +ftp_rmdir +ftp_set_option +ftp_site +ftp_size +ftp_ssl_connect +ftp_systype +ftruncate +func_get_arg +func_get_args +func_num_args +function_exists +fwrite +gc_collect_cycles +gc_disable +gc_enable +gc_enabled +gc_mem_caches +gc_status +gd_info +getSession +get_browser +get_called_class +get_cfg_var +get_class +get_class_methods +get_class_vars +get_current_user +get_debug_type +get_declared_classes +get_declared_interfaces +get_declared_traits +get_defined_constants +get_defined_functions +get_defined_vars +get_extension_funcs +get_headers +get_html_translation_table +get_include_path +get_included_files +get_loaded_extensions +get_magic_quotes_gpc +get_magic_quotes_runtime +get_mangled_object_vars +get_meta_tags +get_object_vars +get_parent_class +get_resource_id +get_resource_type +get_resources +getallheaders +getcwd +getdate +getenv +gethostbyaddr +gethostbyname +gethostbynamel +gethostname +getimagesize +getimagesizefromstring +getlastmod +getmxrr +getmygid +getmyinode +getmypid +getmyuid +getopt +getprotobyname +getprotobynumber +getrandmax +getrusage +getservbyname +getservbyport +gettext +gettimeofday +gettype +glob +gmdate +gmmktime +gmp_abs +gmp_add +gmp_and +gmp_binomial +gmp_clrbit +gmp_cmp +gmp_com +gmp_div_q +gmp_div_qr +gmp_div_r +gmp_divexact +gmp_export +gmp_fact +gmp_gcd +gmp_gcdext +gmp_hamdist +gmp_import +gmp_init +gmp_intval +gmp_invert +gmp_jacobi +gmp_kronecker +gmp_lcm +gmp_legendre +gmp_mod +gmp_mul +gmp_neg +gmp_nextprime +gmp_or +gmp_perfect_power +gmp_perfect_square +gmp_popcount +gmp_pow +gmp_powm +gmp_prob_prime +gmp_random +gmp_random_bits +gmp_random_range +gmp_random_seed +gmp_root +gmp_rootrem +gmp_scan0 +gmp_scan1 +gmp_setbit +gmp_sign +gmp_sqrt +gmp_sqrtrem +gmp_strval +gmp_sub +gmp_testbit +gmp_xor +gmstrftime +grapheme_extract +grapheme_stripos +grapheme_stristr +grapheme_strlen +grapheme_strpos +grapheme_strripos +grapheme_strrpos +grapheme_strstr +grapheme_substr +gregoriantojd +gzclose +gzcompress +gzdecode +gzdeflate +gzencode +gzeof +gzfile +gzgetc +gzgets +gzgetss +gzinflate +gzopen +gzpassthru +gzread +gzrewind +gzseek +gztell +gzuncompress +gzwrite +hash +hash_algos +hash_copy +hash_equals +hash_file +hash_final +hash_hkdf +hash_hmac +hash_hmac_algos +hash_hmac_file +hash_init +hash_pbkdf2 +hash_update +hash_update_file +hash_update_stream +header +header_register_callback +header_remove +headers_list +headers_sent +hebrev +hebrevc +hex2bin +hexdec +highlight_file +highlight_string +hrtime +html_entity_decode +htmlentities +htmlspecialchars +htmlspecialchars_decode +http_build_query +http_response_code +hypot +ibase_add_user +ibase_affected_rows +ibase_backup +ibase_blob_add +ibase_blob_cancel +ibase_blob_close +ibase_blob_create +ibase_blob_echo +ibase_blob_get +ibase_blob_import +ibase_blob_info +ibase_blob_open +ibase_close +ibase_commit +ibase_commit_ret +ibase_connect +ibase_db_info +ibase_delete_user +ibase_drop_db +ibase_errcode +ibase_errmsg +ibase_execute +ibase_fetch_assoc +ibase_fetch_object +ibase_fetch_row +ibase_field_info +ibase_free_event_handler +ibase_free_query +ibase_free_result +ibase_gen_id +ibase_maintain_db +ibase_modify_user +ibase_name_result +ibase_num_fields +ibase_num_params +ibase_param_info +ibase_pconnect +ibase_prepare +ibase_query +ibase_restore +ibase_rollback +ibase_rollback_ret +ibase_server_info +ibase_service_attach +ibase_service_detach +ibase_set_event_handler +ibase_trans +ibase_wait_event +iconv +iconv_get_encoding +iconv_mime_decode +iconv_mime_decode_headers +iconv_mime_encode +iconv_set_encoding +iconv_strlen +iconv_strpos +iconv_strrpos +iconv_substr +idate +idn_to_ascii +idn_to_utf8 +igbinary_serialize +igbinary_unserialize +ignore_user_abort +image2wbmp +image_type_to_extension +image_type_to_mime_type +imageaffine +imageaffinematrixconcat +imageaffinematrixget +imagealphablending +imageantialias +imagearc +imageavif +imagebmp +imagechar +imagecharup +imagecolorallocate +imagecolorallocatealpha +imagecolorat +imagecolorclosest +imagecolorclosestalpha +imagecolorclosesthwb +imagecolordeallocate +imagecolorexact +imagecolorexactalpha +imagecolormatch +imagecolorresolve +imagecolorresolvealpha +imagecolorset +imagecolorsforindex +imagecolorstotal +imagecolortransparent +imageconvolution +imagecopy +imagecopymerge +imagecopymergegray +imagecopyresampled +imagecopyresized +imagecreate +imagecreatefromavif +imagecreatefrombmp +imagecreatefromgd +imagecreatefromgd2 +imagecreatefromgd2part +imagecreatefromgif +imagecreatefromjpeg +imagecreatefrompng +imagecreatefromstring +imagecreatefromtga +imagecreatefromwbmp +imagecreatefromwebp +imagecreatefromxbm +imagecreatefromxpm +imagecreatetruecolor +imagecrop +imagecropauto +imagedashedline +imagedestroy +imageellipse +imagefill +imagefilledarc +imagefilledellipse +imagefilledpolygon +imagefilledrectangle +imagefilltoborder +imagefilter +imageflip +imagefontheight +imagefontwidth +imageftbbox +imagefttext +imagegammacorrect +imagegd +imagegd2 +imagegetclip +imagegetinterpolation +imagegif +imagegrabscreen +imagegrabwindow +imageinterlace +imageistruecolor +imagejpeg +imagelayereffect +imageline +imageloadfont +imageopenpolygon +imagepalettecopy +imagepalettetotruecolor +imagepng +imagepolygon +imagerectangle +imageresolution +imagerotate +imagesavealpha +imagescale +imagesetbrush +imagesetclip +imagesetinterpolation +imagesetpixel +imagesetstyle +imagesetthickness +imagesettile +imagestring +imagestringup +imagesx +imagesy +imagetruecolortopalette +imagettfbbox +imagettftext +imagetypes +imagewbmp +imagewebp +imagexbm +imap_8bit +imap_alerts +imap_append +imap_base64 +imap_binary +imap_body +imap_bodystruct +imap_check +imap_clearflag_full +imap_close +imap_createmailbox +imap_delete +imap_deletemailbox +imap_errors +imap_expunge +imap_fetch_overview +imap_fetchbody +imap_fetchheader +imap_fetchmime +imap_fetchstructure +imap_gc +imap_get_quota +imap_get_quotaroot +imap_getacl +imap_getmailboxes +imap_getsubscribed +imap_headerinfo +imap_headers +imap_last_error +imap_list +imap_listscan +imap_lsub +imap_mail +imap_mail_compose +imap_mail_copy +imap_mail_move +imap_mailboxmsginfo +imap_mime_header_decode +imap_msgno +imap_mutf7_to_utf8 +imap_num_msg +imap_num_recent +imap_open +imap_ping +imap_qprint +imap_renamemailbox +imap_reopen +imap_rfc822_parse_adrlist +imap_rfc822_parse_headers +imap_rfc822_write_address +imap_savebody +imap_search +imap_set_quota +imap_setacl +imap_setflag_full +imap_sort +imap_status +imap_subscribe +imap_thread +imap_timeout +imap_uid +imap_undelete +imap_unsubscribe +imap_utf7_decode +imap_utf7_encode +imap_utf8 +imap_utf8_to_mutf7 +implode +in_array +inet_ntop +inet_pton +inflate_add +inflate_get_read_len +inflate_get_status +inflate_init +ini_get +ini_get_all +ini_restore +ini_set +inotify_add_watch +inotify_init +inotify_queue_len +inotify_read +inotify_rm_watch +intdiv +interface_exists +intl_error_name +intl_get_error_code +intl_get_error_message +intl_is_failure +intval +ip2long +iptcembed +iptcparse +is_a +is_array +is_bool +is_callable +is_countable +is_dir +is_executable +is_file +is_finite +is_float +is_infinite +is_int +is_iterable +is_link +is_nan +is_null +is_numeric +is_object +is_readable +is_resource +is_scalar +is_soap_fault +is_string +is_subclass_of +is_tainted +is_uploaded_file +is_writable +isset +iterator_apply +iterator_count +iterator_to_array +jddayofweek +jdmonthname +jdtofrench +jdtogregorian +jdtojewish +jdtojulian +jdtounix +jewishtojd +jpeg2wbmp +json_decode +json_encode +json_last_error +json_last_error_msg +juliantojd +key +krsort +ksort +lcfirst +lcg_value +lchgrp +lchown +ldap_8859_to_t61 +ldap_add +ldap_add_ext +ldap_bind +ldap_bind_ext +ldap_compare +ldap_connect +ldap_control_paged_result +ldap_control_paged_result_response +ldap_count_entries +ldap_count_references +ldap_delete +ldap_delete_ext +ldap_dn2ufn +ldap_err2str +ldap_errno +ldap_error +ldap_escape +ldap_exop +ldap_exop_passwd +ldap_exop_refresh +ldap_exop_whoami +ldap_explode_dn +ldap_first_attribute +ldap_first_entry +ldap_first_reference +ldap_free_result +ldap_get_attributes +ldap_get_dn +ldap_get_entries +ldap_get_option +ldap_get_values +ldap_get_values_len +ldap_list +ldap_mod_add +ldap_mod_add_ext +ldap_mod_del +ldap_mod_del_ext +ldap_mod_replace +ldap_mod_replace_ext +ldap_modify_batch +ldap_next_attribute +ldap_next_entry +ldap_next_reference +ldap_parse_exop +ldap_parse_reference +ldap_parse_result +ldap_read +ldap_rename +ldap_rename_ext +ldap_sasl_bind +ldap_search +ldap_set_option +ldap_set_rebind_proc +ldap_sort +ldap_start_tls +ldap_t61_to_8859 +ldap_unbind +levenshtein +libxml_clear_errors +libxml_disable_entity_loader +libxml_get_errors +libxml_get_external_entity_loader +libxml_get_last_error +libxml_set_external_entity_loader +libxml_set_streams_context +libxml_use_internal_errors +link +linkinfo +list +localeconv +localtime +log +log10 +log1p +long2ip +lstat +ltrim +lzf_compress +lzf_decompress +lzf_optimized_for +mail +max +mb_check_encoding +mb_chr +mb_convert_case +mb_convert_encoding +mb_convert_kana +mb_convert_variables +mb_decode_mimeheader +mb_decode_numericentity +mb_detect_encoding +mb_detect_order +mb_encode_mimeheader +mb_encode_numericentity +mb_encoding_aliases +mb_ereg +mb_ereg_match +mb_ereg_replace +mb_ereg_replace_callback +mb_ereg_search +mb_ereg_search_getpos +mb_ereg_search_getregs +mb_ereg_search_init +mb_ereg_search_pos +mb_ereg_search_regs +mb_ereg_search_setpos +mb_eregi +mb_eregi_replace +mb_get_info +mb_http_input +mb_http_output +mb_internal_encoding +mb_language +mb_list_encodings +mb_ord +mb_output_handler +mb_parse_str +mb_preferred_mime_name +mb_regex_encoding +mb_regex_set_options +mb_scrub +mb_send_mail +mb_split +mb_str_split +mb_strcut +mb_strimwidth +mb_stripos +mb_stristr +mb_strlen +mb_strpos +mb_strrchr +mb_strrichr +mb_strripos +mb_strrpos +mb_strstr +mb_strtolower +mb_strtoupper +mb_strwidth +mb_substitute_character +mb_substr +mb_substr_count +mcrypt_create_iv +mcrypt_decrypt +mcrypt_enc_get_algorithms_name +mcrypt_enc_get_block_size +mcrypt_enc_get_iv_size +mcrypt_enc_get_key_size +mcrypt_enc_get_modes_name +mcrypt_enc_get_supported_key_sizes +mcrypt_enc_is_block_algorithm +mcrypt_enc_is_block_algorithm_mode +mcrypt_enc_is_block_mode +mcrypt_enc_self_test +mcrypt_encrypt +mcrypt_generic +mcrypt_generic_deinit +mcrypt_generic_init +mcrypt_get_block_size +mcrypt_get_cipher_name +mcrypt_get_iv_size +mcrypt_get_key_size +mcrypt_list_algorithms +mcrypt_list_modes +mcrypt_module_close +mcrypt_module_get_algo_block_size +mcrypt_module_get_algo_key_size +mcrypt_module_get_supported_key_sizes +mcrypt_module_is_block_algorithm +mcrypt_module_is_block_algorithm_mode +mcrypt_module_is_block_mode +mcrypt_module_open +mcrypt_module_self_test +md5 +md5_file +mdecrypt_generic +memcache_debug +memory_get_peak_usage +memory_get_usage +memory_reset_peak_usage +metaphone +method_exists +mhash +mhash_count +mhash_get_block_size +mhash_get_hash_name +mhash_keygen_s2k +microtime +mime_content_type +min +mkdir +mktime +money_format +move_uploaded_file +msg_get_queue +msg_queue_exists +msg_receive +msg_remove_queue +msg_send +msg_set_queue +msg_stat_queue +mt_getrandmax +mt_rand +mt_srand +mysql_affected_rows +mysql_client_encoding +mysql_close +mysql_connect +mysql_create_db +mysql_data_seek +mysql_db_name +mysql_db_query +mysql_drop_db +mysql_errno +mysql_error +mysql_escape_string +mysql_fetch_array +mysql_fetch_assoc +mysql_fetch_field +mysql_fetch_lengths +mysql_fetch_object +mysql_fetch_row +mysql_field_flags +mysql_field_len +mysql_field_name +mysql_field_seek +mysql_field_table +mysql_field_type +mysql_free_result +mysql_get_client_info +mysql_get_host_info +mysql_get_proto_info +mysql_get_server_info +mysql_info +mysql_insert_id +mysql_list_dbs +mysql_list_fields +mysql_list_processes +mysql_list_tables +mysql_num_fields +mysql_num_rows +mysql_pconnect +mysql_ping +mysql_query +mysql_real_escape_string +mysql_result +mysql_select_db +mysql_set_charset +mysql_stat +mysql_tablename +mysql_thread_id +mysql_unbuffered_query +mysqli_get_client_stats +mysqli_get_links_stats +natcasesort +natsort +net_get_interfaces +next +ngettext +nl2br +nl_langinfo +number_format +oauth_get_sbs +oauth_urlencode +ob_clean +ob_end_clean +ob_end_flush +ob_flush +ob_get_clean +ob_get_contents +ob_get_flush +ob_get_length +ob_get_level +ob_get_status +ob_gzhandler +ob_iconv_handler +ob_implicit_flush +ob_list_handlers +ob_start +ob_tidyhandler +oci_bind_array_by_name +oci_bind_by_name +oci_cancel +oci_client_version +oci_close +oci_commit +oci_connect +oci_define_by_name +oci_error +oci_execute +oci_fetch +oci_fetch_all +oci_fetch_array +oci_fetch_assoc +oci_fetch_object +oci_fetch_row +oci_field_is_null +oci_field_name +oci_field_precision +oci_field_scale +oci_field_size +oci_field_type +oci_field_type_raw +oci_free_descriptor +oci_free_statement +oci_get_implicit_resultset +oci_internal_debug +oci_lob_copy +oci_lob_is_equal +oci_new_collection +oci_new_connect +oci_new_cursor +oci_new_descriptor +oci_num_fields +oci_num_rows +oci_parse +oci_password_change +oci_pconnect +oci_register_taf_callback +oci_result +oci_rollback +oci_server_version +oci_set_action +oci_set_call_timeout +oci_set_client_identifier +oci_set_client_info +oci_set_db_operation +oci_set_edition +oci_set_module_name +oci_set_prefetch +oci_set_prefetch_lob +oci_statement_type +oci_unregister_taf_callback +ocifetchinto +octdec +odbc_autocommit +odbc_binmode +odbc_close +odbc_close_all +odbc_columnprivileges +odbc_columns +odbc_commit +odbc_connect +odbc_cursor +odbc_data_source +odbc_error +odbc_errormsg +odbc_exec +odbc_execute +odbc_fetch_array +odbc_fetch_into +odbc_fetch_object +odbc_fetch_row +odbc_field_len +odbc_field_name +odbc_field_num +odbc_field_scale +odbc_field_type +odbc_foreignkeys +odbc_free_result +odbc_gettypeinfo +odbc_longreadlen +odbc_next_result +odbc_num_fields +odbc_num_rows +odbc_pconnect +odbc_prepare +odbc_primarykeys +odbc_procedurecolumns +odbc_procedures +odbc_result +odbc_result_all +odbc_rollback +odbc_setoption +odbc_specialcolumns +odbc_statistics +odbc_tableprivileges +odbc_tables +opcache_compile_file +opcache_get_configuration +opcache_get_status +opcache_invalidate +opcache_is_script_cached +opcache_reset +opendir +openlog +openssl_cipher_iv_length +openssl_cipher_key_length +openssl_cms_decrypt +openssl_cms_encrypt +openssl_cms_read +openssl_cms_sign +openssl_cms_verify +openssl_csr_export +openssl_csr_export_to_file +openssl_csr_get_public_key +openssl_csr_get_subject +openssl_csr_new +openssl_csr_sign +openssl_decrypt +openssl_dh_compute_key +openssl_digest +openssl_encrypt +openssl_error_string +openssl_free_key +openssl_get_cert_locations +openssl_get_cipher_methods +openssl_get_curve_names +openssl_get_md_methods +openssl_open +openssl_pbkdf2 +openssl_pkcs12_export +openssl_pkcs12_export_to_file +openssl_pkcs12_read +openssl_pkcs7_decrypt +openssl_pkcs7_encrypt +openssl_pkcs7_read +openssl_pkcs7_sign +openssl_pkcs7_verify +openssl_pkey_derive +openssl_pkey_export +openssl_pkey_export_to_file +openssl_pkey_free +openssl_pkey_get_details +openssl_pkey_get_private +openssl_pkey_get_public +openssl_pkey_new +openssl_private_decrypt +openssl_private_encrypt +openssl_public_decrypt +openssl_public_encrypt +openssl_random_pseudo_bytes +openssl_seal +openssl_sign +openssl_spki_export +openssl_spki_export_challenge +openssl_spki_new +openssl_spki_verify +openssl_verify +openssl_x509_check_private_key +openssl_x509_checkpurpose +openssl_x509_export +openssl_x509_export_to_file +openssl_x509_fingerprint +openssl_x509_free +openssl_x509_parse +openssl_x509_read +openssl_x509_verify +ord +output_add_rewrite_var +output_reset_rewrite_vars +pack +parse_ini_file +parse_ini_string +parse_str +parse_url +passthru +password_algos +password_get_info +password_hash +password_needs_rehash +password_verify +pathinfo +pclose +pcntl_alarm +pcntl_async_signals +pcntl_exec +pcntl_fork +pcntl_get_last_error +pcntl_getpriority +pcntl_rfork +pcntl_setpriority +pcntl_signal +pcntl_signal_dispatch +pcntl_signal_get_handler +pcntl_sigprocmask +pcntl_sigtimedwait +pcntl_sigwaitinfo +pcntl_strerror +pcntl_unshare +pcntl_wait +pcntl_waitpid +pcntl_wexitstatus +pcntl_wifexited +pcntl_wifsignaled +pcntl_wifstopped +pcntl_wstopsig +pcntl_wtermsig +pfsockopen +pg_affected_rows +pg_cancel_query +pg_client_encoding +pg_close +pg_connect +pg_connect_poll +pg_connection_busy +pg_connection_reset +pg_connection_status +pg_consume_input +pg_convert +pg_copy_from +pg_copy_to +pg_dbname +pg_delete +pg_end_copy +pg_escape_bytea +pg_escape_identifier +pg_escape_literal +pg_escape_string +pg_execute +pg_fetch_all +pg_fetch_all_columns +pg_fetch_array +pg_fetch_assoc +pg_fetch_object +pg_fetch_result +pg_fetch_row +pg_field_is_null +pg_field_name +pg_field_num +pg_field_prtlen +pg_field_size +pg_field_table +pg_field_type +pg_field_type_oid +pg_flush +pg_free_result +pg_get_notify +pg_get_pid +pg_get_result +pg_host +pg_insert +pg_last_error +pg_last_notice +pg_last_oid +pg_lo_close +pg_lo_create +pg_lo_export +pg_lo_import +pg_lo_open +pg_lo_read +pg_lo_read_all +pg_lo_seek +pg_lo_tell +pg_lo_truncate +pg_lo_unlink +pg_lo_write +pg_meta_data +pg_num_fields +pg_num_rows +pg_options +pg_parameter_status +pg_pconnect +pg_ping +pg_port +pg_prepare +pg_put_line +pg_query +pg_query_params +pg_result_error +pg_result_error_field +pg_result_seek +pg_result_status +pg_select +pg_send_execute +pg_send_prepare +pg_send_query +pg_send_query_params +pg_set_client_encoding +pg_set_error_verbosity +pg_socket +pg_trace +pg_transaction_status +pg_tty +pg_unescape_bytea +pg_untrace +pg_update +pg_version +php_ini_loaded_file +php_ini_scanned_files +php_sapi_name +php_strip_whitespace +php_uname +phpcredits +phpdbg_break_file +phpdbg_break_function +phpdbg_break_method +phpdbg_break_next +phpdbg_clear +phpdbg_color +phpdbg_end_oplog +phpdbg_exec +phpdbg_get_executable +phpdbg_prompt +phpdbg_start_oplog +phpinfo +phpversion +pi +png2wbmp +popen +posix_access +posix_ctermid +posix_get_last_error +posix_getcwd +posix_getegid +posix_geteuid +posix_getgid +posix_getgrgid +posix_getgrnam +posix_getgroups +posix_getlogin +posix_getpgid +posix_getpgrp +posix_getpid +posix_getppid +posix_getpwnam +posix_getpwuid +posix_getrlimit +posix_getsid +posix_getuid +posix_initgroups +posix_isatty +posix_kill +posix_mkfifo +posix_mknod +posix_setegid +posix_seteuid +posix_setgid +posix_setpgid +posix_setrlimit +posix_setsid +posix_setuid +posix_strerror +posix_times +posix_ttyname +posix_uname +pow +preg_filter +preg_grep +preg_last_error +preg_last_error_msg +preg_match +preg_match_all +preg_quote +preg_replace +preg_replace_callback +preg_replace_callback_array +preg_split +prev +print +print_r +printf +proc_close +proc_get_status +proc_nice +proc_open +proc_terminate +property_exists +pspell_add_to_personal +pspell_add_to_session +pspell_check +pspell_clear_session +pspell_config_create +pspell_config_data_dir +pspell_config_dict_dir +pspell_config_ignore +pspell_config_mode +pspell_config_personal +pspell_config_repl +pspell_config_runtogether +pspell_config_save_repl +pspell_new +pspell_new_config +pspell_new_personal +pspell_save_wordlist +pspell_store_replacement +pspell_suggest +putenv +quoted_printable_decode +quoted_printable_encode +quotemeta +rad2deg +rand +random_bytes +random_int +range +rar_wrapper_cache_stats +rawurldecode +rawurlencode +readdir +readfile +readgzfile +readline +readline_add_history +readline_callback_handler_install +readline_callback_handler_remove +readline_callback_read_char +readline_clear_history +readline_completion_function +readline_info +readline_list_history +readline_on_new_line +readline_read_history +readline_redisplay +readline_write_history +readlink +realpath +realpath_cache_get +realpath_cache_size +recode_file +recode_string +register_shutdown_function +register_tick_function +rename +reset +restore_error_handler +restore_exception_handler +restore_include_path +rewind +rewinddir +rmdir +round +rpmvercmp +rrd_create +rrdc_disconnect +rsort +rtrim +sapi_windows_cp_conv +sapi_windows_cp_get +sapi_windows_cp_is_utf8 +sapi_windows_cp_set +sapi_windows_generate_ctrl_event +sapi_windows_set_ctrl_handler +sapi_windows_vt100_support +scandir +scoutapm_get_calls +scoutapm_list_instrumented_functions +seaslog_get_author +seaslog_get_version +sem_acquire +sem_get +sem_release +sem_remove +serialize +session_abort +session_cache_expire +session_cache_limiter +session_create_id +session_decode +session_destroy +session_encode +session_gc +session_get_cookie_params +session_id +session_module_name +session_name +session_regenerate_id +session_register_shutdown +session_reset +session_save_path +session_set_cookie_params +session_set_save_handler +session_start +session_status +session_unset +session_write_close +set_error_handler +set_exception_handler +set_include_path +set_time_limit +setcookie +setlocale +setrawcookie +settype +sha1 +sha1_file +shell_exec +shm_attach +shm_detach +shm_get_var +shm_has_var +shm_put_var +shm_remove +shm_remove_var +shmop_close +shmop_delete +shmop_open +shmop_read +shmop_size +shmop_write +shuffle +similar_text +simplexml_import_dom +simplexml_load_file +simplexml_load_string +sin +sinh +sleep +snmp2_get +snmp2_getnext +snmp2_real_walk +snmp2_set +snmp2_walk +snmp3_get +snmp3_getnext +snmp3_real_walk +snmp3_set +snmp3_walk +snmp_get_quick_print +snmp_get_valueretrieval +snmp_read_mib +snmp_set_enum_print +snmp_set_oid_output_format +snmp_set_quick_print +snmp_set_valueretrieval +snmpget +snmpgetnext +snmprealwalk +snmpset +snmpwalk +snmpwalkoid +socket_accept +socket_addrinfo_bind +socket_addrinfo_connect +socket_addrinfo_explain +socket_addrinfo_lookup +socket_bind +socket_clear_error +socket_close +socket_cmsg_space +socket_connect +socket_create +socket_create_listen +socket_create_pair +socket_export_stream +socket_get_option +socket_getpeername +socket_getsockname +socket_import_stream +socket_last_error +socket_listen +socket_read +socket_recv +socket_recvfrom +socket_recvmsg +socket_select +socket_send +socket_sendmsg +socket_sendto +socket_set_block +socket_set_nonblock +socket_set_option +socket_shutdown +socket_strerror +socket_write +socket_wsaprotocol_info_export +socket_wsaprotocol_info_import +socket_wsaprotocol_info_release +solr_get_version +sort +soundex +spl_autoload +spl_autoload_call +spl_autoload_extensions +spl_autoload_functions +spl_autoload_register +spl_autoload_unregister +spl_classes +spl_object_hash +spl_object_id +sprintf +sqrt +srand +sscanf +ssdeep_fuzzy_compare +ssdeep_fuzzy_hash +ssdeep_fuzzy_hash_filename +stat +stomp_connect_error +stomp_version +str_contains +str_ends_with +str_getcsv +str_ireplace +str_pad +str_repeat +str_replace +str_rot13 +str_shuffle +str_split +str_starts_with +str_word_count +strcasecmp +strcmp +strcoll +strcspn +stream_bucket_append +stream_bucket_make_writeable +stream_bucket_new +stream_bucket_prepend +stream_context_create +stream_context_get_default +stream_context_get_options +stream_context_get_params +stream_context_set_default +stream_context_set_option +stream_context_set_params +stream_copy_to_stream +stream_filter_append +stream_filter_prepend +stream_filter_register +stream_filter_remove +stream_get_contents +stream_get_filters +stream_get_line +stream_get_meta_data +stream_get_transports +stream_get_wrappers +stream_is_local +stream_isatty +stream_notification_callback +stream_resolve_include_path +stream_select +stream_set_blocking +stream_set_chunk_size +stream_set_read_buffer +stream_set_timeout +stream_set_write_buffer +stream_socket_accept +stream_socket_client +stream_socket_enable_crypto +stream_socket_get_name +stream_socket_pair +stream_socket_recvfrom +stream_socket_sendto +stream_socket_server +stream_socket_shutdown +stream_supports_lock +stream_wrapper_register +stream_wrapper_restore +stream_wrapper_unregister +strftime +strip_tags +stripcslashes +stripos +stripslashes +stristr +strlen +strnatcasecmp +strnatcmp +strncasecmp +strncmp +strpbrk +strpos +strptime +strrchr +strrev +strripos +strrpos +strspn +strstr +strtok +strtolower +strtotime +strtoupper +strtr +strval +substr +substr_compare +substr_count +substr_replace +symlink +sys_get_temp_dir +sys_getloadavg +syslog +system +taint +tan +tanh +tcpwrap_check +tempnam +textdomain +tidy_access_count +tidy_config_count +tidy_error_count +tidy_get_output +tidy_warning_count +time +time_nanosleep +time_sleep_until +timezone_name_from_abbr +timezone_version_get +tmpfile +token_get_all +token_name +touch +trader_acos +trader_ad +trader_add +trader_adosc +trader_adx +trader_adxr +trader_apo +trader_aroon +trader_aroonosc +trader_asin +trader_atan +trader_atr +trader_avgprice +trader_bbands +trader_beta +trader_bop +trader_cci +trader_cdl2crows +trader_cdl3blackcrows +trader_cdl3inside +trader_cdl3linestrike +trader_cdl3outside +trader_cdl3starsinsouth +trader_cdl3whitesoldiers +trader_cdlabandonedbaby +trader_cdladvanceblock +trader_cdlbelthold +trader_cdlbreakaway +trader_cdlclosingmarubozu +trader_cdlconcealbabyswall +trader_cdlcounterattack +trader_cdldarkcloudcover +trader_cdldoji +trader_cdldojistar +trader_cdldragonflydoji +trader_cdlengulfing +trader_cdleveningdojistar +trader_cdleveningstar +trader_cdlgapsidesidewhite +trader_cdlgravestonedoji +trader_cdlhammer +trader_cdlhangingman +trader_cdlharami +trader_cdlharamicross +trader_cdlhighwave +trader_cdlhikkake +trader_cdlhikkakemod +trader_cdlhomingpigeon +trader_cdlidentical3crows +trader_cdlinneck +trader_cdlinvertedhammer +trader_cdlkicking +trader_cdlkickingbylength +trader_cdlladderbottom +trader_cdllongleggeddoji +trader_cdllongline +trader_cdlmarubozu +trader_cdlmatchinglow +trader_cdlmathold +trader_cdlmorningdojistar +trader_cdlmorningstar +trader_cdlonneck +trader_cdlpiercing +trader_cdlrickshawman +trader_cdlrisefall3methods +trader_cdlseparatinglines +trader_cdlshootingstar +trader_cdlshortline +trader_cdlspinningtop +trader_cdlstalledpattern +trader_cdlsticksandwich +trader_cdltakuri +trader_cdltasukigap +trader_cdlthrusting +trader_cdltristar +trader_cdlunique3river +trader_cdlupsidegap2crows +trader_cdlxsidegap3methods +trader_ceil +trader_cmo +trader_correl +trader_cos +trader_cosh +trader_dema +trader_div +trader_dx +trader_ema +trader_errno +trader_exp +trader_floor +trader_get_compat +trader_get_unstable_period +trader_ht_dcperiod +trader_ht_dcphase +trader_ht_phasor +trader_ht_sine +trader_ht_trendline +trader_ht_trendmode +trader_kama +trader_linearreg +trader_linearreg_angle +trader_linearreg_intercept +trader_linearreg_slope +trader_ln +trader_log10 +trader_ma +trader_macd +trader_macdext +trader_macdfix +trader_mama +trader_mavp +trader_max +trader_maxindex +trader_medprice +trader_mfi +trader_midpoint +trader_midprice +trader_min +trader_minindex +trader_minmax +trader_minmaxindex +trader_minus_di +trader_minus_dm +trader_mom +trader_mult +trader_natr +trader_obv +trader_plus_di +trader_plus_dm +trader_ppo +trader_roc +trader_rocp +trader_rocr +trader_rocr100 +trader_rsi +trader_sar +trader_sarext +trader_set_compat +trader_set_unstable_period +trader_sin +trader_sinh +trader_sma +trader_sqrt +trader_stddev +trader_stoch +trader_stochf +trader_stochrsi +trader_sub +trader_sum +trader_t3 +trader_tan +trader_tanh +trader_tema +trader_trange +trader_trima +trader_trix +trader_tsf +trader_typprice +trader_ultosc +trader_var +trader_wclprice +trader_willr +trader_wma +trait_exists +trigger_error +trim +uasort +ucfirst +ucwords +uksort +umask +uniqid +unixtojd +unlink +unpack +unregister_tick_function +unserialize +unset +untaint +urldecode +urlencode +use_soap_error_handler +usleep +usort +utf8_decode +utf8_encode +var_dump +var_export +var_representation +variant_abs +variant_add +variant_and +variant_cast +variant_cat +variant_cmp +variant_date_from_timestamp +variant_date_to_timestamp +variant_div +variant_eqv +variant_fix +variant_get_type +variant_idiv +variant_imp +variant_int +variant_mod +variant_mul +variant_neg +variant_not +variant_or +variant_pow +variant_round +variant_set +variant_set_type +variant_sub +variant_xor +version_compare +vfprintf +virtual +vprintf +vsprintf +wddx_add_vars +wddx_deserialize +wddx_packet_end +wddx_packet_start +wddx_serialize_value +wddx_serialize_vars +win32_continue_service +win32_create_service +win32_delete_service +win32_get_last_control_message +win32_pause_service +win32_query_service_status +win32_send_custom_control +win32_set_service_exit_code +win32_set_service_exit_mode +win32_set_service_status +win32_start_service +win32_start_service_ctrl_dispatcher +win32_stop_service +wincache_fcache_fileinfo +wincache_fcache_meminfo +wincache_lock +wincache_ocache_fileinfo +wincache_ocache_meminfo +wincache_refresh_if_changed +wincache_rplist_fileinfo +wincache_rplist_meminfo +wincache_scache_info +wincache_scache_meminfo +wincache_ucache_add +wincache_ucache_cas +wincache_ucache_clear +wincache_ucache_dec +wincache_ucache_delete +wincache_ucache_exists +wincache_ucache_get +wincache_ucache_inc +wincache_ucache_info +wincache_ucache_meminfo +wincache_ucache_set +wincache_unlock +wordwrap +xattr_get +xattr_list +xattr_remove +xattr_set +xattr_supported +xdiff_file_bdiff +xdiff_file_bdiff_size +xdiff_file_bpatch +xdiff_file_diff +xdiff_file_diff_binary +xdiff_file_merge3 +xdiff_file_patch +xdiff_file_patch_binary +xdiff_file_rabdiff +xdiff_string_bdiff +xdiff_string_bdiff_size +xdiff_string_bpatch +xdiff_string_diff +xdiff_string_diff_binary +xdiff_string_merge3 +xdiff_string_patch +xdiff_string_patch_binary +xdiff_string_rabdiff +xhprof_disable +xhprof_enable +xhprof_sample_disable +xhprof_sample_enable +xml_error_string +xml_get_current_byte_index +xml_get_current_column_number +xml_get_current_line_number +xml_get_error_code +xml_parse +xml_parse_into_struct +xml_parser_create +xml_parser_create_ns +xml_parser_free +xml_parser_get_option +xml_parser_set_option +xml_set_character_data_handler +xml_set_default_handler +xml_set_element_handler +xml_set_end_namespace_decl_handler +xml_set_external_entity_ref_handler +xml_set_notation_decl_handler +xml_set_object +xml_set_processing_instruction_handler +xml_set_start_namespace_decl_handler +xml_set_unparsed_entity_decl_handler +xmlrpc_decode +xmlrpc_decode_request +xmlrpc_encode +xmlrpc_encode_request +xmlrpc_get_type +xmlrpc_is_fault +xmlrpc_parse_method_descriptions +xmlrpc_server_add_introspection_data +xmlrpc_server_call_method +xmlrpc_server_create +xmlrpc_server_destroy +xmlrpc_server_register_introspection_callback +xmlrpc_server_register_method +xmlrpc_set_type +yaml_emit +yaml_emit_file +yaml_parse +yaml_parse_file +yaml_parse_url +yaz_addinfo +yaz_ccl_conf +yaz_ccl_parse +yaz_close +yaz_connect +yaz_database +yaz_element +yaz_errno +yaz_error +yaz_es +yaz_es_result +yaz_get_option +yaz_hits +yaz_itemorder +yaz_present +yaz_range +yaz_record +yaz_scan +yaz_scan_result +yaz_schema +yaz_search +yaz_set_option +yaz_sort +yaz_syntax +yaz_wait +zend_thread_id +zend_version +zip_close +zip_entry_close +zip_entry_compressedsize +zip_entry_compressionmethod +zip_entry_filesize +zip_entry_name +zip_entry_open +zip_entry_read +zip_open +zip_read +zlib_decode +zlib_encode +zlib_get_coding_type +zookeeper_dispatch +__autoload +__halt_compiler diff --git a/platform/frontends/php2atom/src/main/resources/known_function_signatures.txt b/platform/frontends/php2atom/src/main/resources/known_function_signatures.txt new file mode 100644 index 00000000..a0c9aad5 --- /dev/null +++ b/platform/frontends/php2atom/src/main/resources/known_function_signatures.txt @@ -0,0 +1,56 @@ +// function name; r1, r2; p1_t1, p1_t2; p2_t1; ... +add_post_meta; int, bool; int; string; mixed; bool +apply_filters; mixed; string; mixed; mixed +array_map; array; callable, null; array; array; array +array_merge; array; array; array; array +array_walk_recursive; bool; array, object; callable; mixed +base64_decode; string; string; bool +base64_encode; string; string +count; int; array, countable; int +current; mixed; array, object +do_action; ; string; mixed; +echo; void; string +empty; bool; mixed +explode; array; string; string; int +floatval; float; mixed +in_array; bool; mixed; array; bool +intval; int; mixed +is_array; bool; mixed +is_bool; bool; mixed +is_double; bool; mixed +is_float; bool; mixed +is_int; bool; mixed +is_integer; bool; mixed +is_iterable; bool; mixed +is_long; bool; mixed +is_null; bool; mixed +is_numeric; bool; mixed +is_object; bool; mixed +is_real; bool; mixed +is_resource; bool; mixed +is_scalar; bool; mixed +is_string; bool; mixed +isset; bool; mixed; array; bool +list; array; mixed; mixed; mixed; mixed +maybe_unserialize; mixed; string +number_format; string; float; int; string, null; string, null +preg_match; int, bool; string; string; array; int; int +preg_match_all; int, bool; string; string; array; int; int +preg_replace; string, array, null; string, array; string, array; string, array; int; int +printf; int; string; mixed; mixed; mixed; mixed +rawurldecode; string; string +rtrim; string; string; string +selected; string; mixed; mixed; bool +serialize; string; mixed +sort; bool; array; int +sprintf; string; string; mixed +strip_tags; string; string; array, string, null +strpos; int, bool; string; string; int +strtolower; string; string +strtotime; int, bool; string; int, null +substr; string; string; int; int, null +trim; string; string; string +unserialize; mixed; string; array +urldecode; string; string +var_dump; ; mixed; mixed +wp_json_encode; string,bool; mixed; int; int \ No newline at end of file diff --git a/platform/frontends/php2atom/src/main/resources/log4j2.xml b/platform/frontends/php2atom/src/main/resources/log4j2.xml new file mode 100644 index 00000000..ab3b2d88 --- /dev/null +++ b/platform/frontends/php2atom/src/main/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/platform/frontends/php2atom/src/main/resources/php.ini b/platform/frontends/php2atom/src/main/resources/php.ini new file mode 100644 index 00000000..e1a8f742 --- /dev/null +++ b/platform/frontends/php2atom/src/main/resources/php.ini @@ -0,0 +1,2 @@ +; Uncap memory to avoid OOM errors when parsing large files +memory_limit = -1 \ No newline at end of file diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/Main.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/Main.scala new file mode 100644 index 00000000..40f51f4f --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/Main.scala @@ -0,0 +1,39 @@ +package io.appthreat.php2atom + +import io.appthreat.x2cpg.{X2CpgConfig, X2CpgMain} +import io.appthreat.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery} +import io.appthreat.php2atom.Frontend.* +import scopt.OParser + +/** Command line configuration parameters + */ +final case class Config(phpIni: Option[String] = None, phpParserBin: Option[String] = None) + extends X2CpgConfig[Config] + with TypeRecoveryParserConfig[Config]: + def withPhpIni(phpIni: String): Config = + copy(phpIni = Some(phpIni)).withInheritedFields(this) + + def withPhpParserBin(phpParserBin: String): Config = + copy(phpParserBin = Some(phpParserBin)).withInheritedFields(this) + +object Frontend: + + implicit val defaultConfig: Config = Config() + + val cmdLineParser: OParser[Unit, Config] = + val builder = OParser.builder[Config] + import builder.* + OParser.sequence( + programName("php2atom"), + opt[String]("php-ini") + .action((x, c) => c.withPhpIni(x)) + .text("php.ini path used by php-parser. Defaults to php.ini shipped with Chen."), + opt[String]("php-parser-bin") + .action((x, c) => c.withPhpParserBin(x)) + .text("path to php-parser.phar binary."), + XTypeRecovery.parserOptions + ) + +object Main extends X2CpgMain(cmdLineParser, new Php2Atom()): + def run(config: Config, php2Cpg: Php2Atom): Unit = + php2Cpg.run(config) diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/Php2Atom.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/Php2Atom.scala new file mode 100644 index 00000000..8358c898 --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/Php2Atom.scala @@ -0,0 +1,69 @@ +package io.appthreat.php2atom + +import io.appthreat.php2atom.parser.PhpParser +import io.appthreat.php2atom.passes.* +import io.appthreat.x2cpg.X2Cpg.withNewEmptyCpg +import io.appthreat.x2cpg.X2CpgFrontend +import io.appthreat.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass} +import io.appthreat.x2cpg.utils.ExternalCommand +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.Languages +import io.shiftleft.passes.CpgPassBase +import org.slf4j.LoggerFactory + +import scala.collection.mutable +import scala.util.{Failure, Success, Try} + +class Php2Atom extends X2CpgFrontend[Config]: + private val logger = LoggerFactory.getLogger(this.getClass) + + private def isPhpVersionSupported: Boolean = + val result = ExternalCommand.run("php --version", ".") + result match + case Success(listString) => + true + case Failure(exception) => + logger.debug(s"Failed to run php --version: ${exception.getMessage}") + false + + override def createCpg(config: Config): Try[Cpg] = + val errorMessages = mutable.ListBuffer[String]() + + val parser = PhpParser.getParser(config) + + if parser.isEmpty then + errorMessages.append("Could not initialize PhpParser") + if !isPhpVersionSupported then + errorMessages.append( + "PHP version not supported. Is PHP 7.1.0 or above installed and available on your path?" + ) + + if errorMessages.isEmpty then + withNewEmptyCpg(config.outputPath, config: Config) { (cpg, config) => + new MetaDataPass(cpg, Languages.PHP, config.inputPath).createAndApply() + new AstCreationPass(config, cpg, parser.get)( + config.schemaValidation + ).createAndApply() + new AstParentInfoPass(cpg).createAndApply() + new AnyTypePass(cpg).createAndApply() + TypeNodePass.withTypesFromCpg(cpg).createAndApply() + LocalCreationPass.allLocalCreationPasses(cpg).foreach(_.createAndApply()) + new ClosureRefPass(cpg).createAndApply() + } + else + val errorOutput = ( + "Skipping AST creation as php/php-parser could not be executed." :: + errorMessages.toList + ).mkString("\n- ") + + logger.error(errorOutput) + + Failure(new RuntimeException("php not found or version not supported")) + end if + end createCpg +end Php2Atom + +object Php2Atom: + + def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] = + List(new PhpSetKnownTypesPass(cpg)) diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/astcreation/AstCreator.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/astcreation/AstCreator.scala new file mode 100644 index 00000000..3a789caf --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/astcreation/AstCreator.scala @@ -0,0 +1,1860 @@ +package io.appthreat.php2atom.astcreation + +import io.appthreat.php2atom.astcreation.AstCreator.{NameConstants, TypeConstants, operatorSymbols} +import io.appthreat.php2atom.datastructures.ArrayIndexTracker +import io.appthreat.php2atom.parser.Domain.* +import io.appthreat.php2atom.parser.Domain.PhpModifiers.containsAccessModifier +import io.appthreat.php2atom.utils.Scope +import io.appthreat.x2cpg.Ast.storeInDiffGraph +import io.appthreat.x2cpg.Defines.{StaticInitMethodName, UnresolvedNamespace, UnresolvedSignature} +import io.appthreat.x2cpg.utils.AstPropertiesUtil.RootProperties +import io.appthreat.x2cpg.utils.NodeBuilders.* +import io.appthreat.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, ValidationMode} +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.passes.IntervalKeyPool +import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal +import org.slf4j.LoggerFactory +import overflowdb.BatchedUpdate + +class AstCreator(filename: String, phpAst: PhpFile)(implicit withSchemaValidation: ValidationMode) + extends AstCreatorBase(filename) + with AstNodeBuilder[PhpNode, AstCreator]: + + private val logger = LoggerFactory.getLogger(AstCreator.getClass) + private val scope = new Scope()(() => nextClosureName()) + private val tmpKeyPool = new IntervalKeyPool(first = 0, last = Long.MaxValue) + private val globalNamespace = globalNamespaceBlock() + + private def getNewTmpName(prefix: String = "tmp"): String = + s"$prefix${tmpKeyPool.next.toString}" + + override def createAst(): BatchedUpdate.DiffGraphBuilder = + val ast = astForPhpFile(phpAst) + storeInDiffGraph(ast, diffGraph) + diffGraph + + private def flattenGlobalNamespaceStmt(stmt: PhpStmt): List[PhpStmt] = + stmt match + case namespace: PhpNamespaceStmt if namespace.name.isEmpty => + namespace.stmts + + case _ => stmt :: Nil + + private def globalTypeDeclNode(file: PhpFile, globalNamespace: NewNamespaceBlock): NewTypeDecl = + typeDeclNode( + file, + globalNamespace.name, + globalNamespace.fullName, + filename, + globalNamespace.code, + NodeTypes.NAMESPACE_BLOCK, + globalNamespace.fullName + ) + + private def globalMethodDeclStmt(file: PhpFile, bodyStmts: List[PhpStmt]): PhpMethodDecl = + val modifiersList = List(ModifierTypes.VIRTUAL, ModifierTypes.PUBLIC, ModifierTypes.STATIC) + PhpMethodDecl( + name = PhpNameExpr(NamespaceTraversal.globalNamespaceName, file.attributes), + params = Nil, + modifiers = modifiersList, + returnType = None, + stmts = bodyStmts, + returnByRef = false, + namespacedName = None, + isClassMethod = false, + attributes = file.attributes + ) + + private def astForPhpFile(file: PhpFile): Ast = + scope.pushNewScope(globalNamespace) + + val (globalDeclStmts, globalMethodStmts) = + file.children.flatMap(flattenGlobalNamespaceStmt).partition( + _.isInstanceOf[PhpConstStmt] + ) + + val globalMethodStmt = globalMethodDeclStmt(file, globalMethodStmts) + + val globalTypeDeclStmt = PhpClassLikeStmt( + name = Some(PhpNameExpr(globalNamespace.name, file.attributes)), + modifiers = Nil, + extendsNames = Nil, + implementedInterfaces = Nil, + stmts = globalDeclStmts.appended(globalMethodStmt), + classLikeType = ClassLikeTypes.Class, + scalarType = None, + hasConstructor = false, + attributes = file.attributes + ) + + val globalTypeDeclAst = astForClassLikeStmt(globalTypeDeclStmt) + + scope.popScope() // globalNamespace + + Ast(globalNamespace).withChild(globalTypeDeclAst) + end astForPhpFile + + private def astsForStmt(stmt: PhpStmt): List[Ast] = + stmt match + case echoStmt: PhpEchoStmt => astForEchoStmt(echoStmt) :: Nil + case methodDecl: PhpMethodDecl => astForMethodDecl(methodDecl) :: Nil + case expr: PhpExpr => astForExpr(expr) :: Nil + case breakStmt: PhpBreakStmt => astForBreakStmt(breakStmt) :: Nil + case contStmt: PhpContinueStmt => astForContinueStmt(contStmt) :: Nil + case whileStmt: PhpWhileStmt => astForWhileStmt(whileStmt) :: Nil + case doStmt: PhpDoStmt => astForDoStmt(doStmt) :: Nil + case forStmt: PhpForStmt => astForForStmt(forStmt) :: Nil + case ifStmt: PhpIfStmt => astForIfStmt(ifStmt) :: Nil + case switchStmt: PhpSwitchStmt => astForSwitchStmt(switchStmt) :: Nil + case tryStmt: PhpTryStmt => astForTryStmt(tryStmt) :: Nil + case returnStmt: PhpReturnStmt => astForReturnStmt(returnStmt) :: Nil + case classLikeStmt: PhpClassLikeStmt => astForClassLikeStmt(classLikeStmt) :: Nil + case gotoStmt: PhpGotoStmt => astForGotoStmt(gotoStmt) :: Nil + case labelStmt: PhpLabelStmt => astForLabelStmt(labelStmt) :: Nil + case namespace: PhpNamespaceStmt => astForNamespaceStmt(namespace) :: Nil + case declareStmt: PhpDeclareStmt => astForDeclareStmt(declareStmt) :: Nil + case _: NopStmt => Nil // TODO This'll need to be updated when comments are added. + case haltStmt: PhpHaltCompilerStmt => astForHaltCompilerStmt(haltStmt) :: Nil + case unsetStmt: PhpUnsetStmt => astForUnsetStmt(unsetStmt) :: Nil + case globalStmt: PhpGlobalStmt => astForGlobalStmt(globalStmt) :: Nil + case useStmt: PhpUseStmt => astForUseStmt(useStmt) :: Nil + case groupUseStmt: PhpGroupUseStmt => astForGroupUseStmt(groupUseStmt) :: Nil + case foreachStmt: PhpForeachStmt => astForForeachStmt(foreachStmt) :: Nil + case traitUseStmt: PhpTraitUseStmt => astforTraitUseStmt(traitUseStmt) :: Nil + case enumCase: PhpEnumCaseStmt => astForEnumCase(enumCase) :: Nil + case staticStmt: PhpStaticStmt => astsForStaticStmt(staticStmt) + case unhandled => + logger.debug(s"Unhandled stmt $unhandled in $filename") + ??? + + private def astForEchoStmt(echoStmt: PhpEchoStmt): Ast = + val args = echoStmt.exprs.map(astForExpr) + val code = s"echo ${args.map(_.rootCodeOrEmpty).mkString(",")}" + val callNode = newOperatorCallNode("echo", code, line = line(echoStmt)) + callAst(callNode, args) + + private def thisParamAstForMethod(originNode: PhpNode): Ast = + val typeFullName = scope.getEnclosingTypeDeclTypeFullName.getOrElse(TypeConstants.Any) + + val thisNode = parameterInNode( + originNode, + name = NameConstants.This, + code = NameConstants.This, + index = 0, + isVariadic = false, + evaluationStrategy = EvaluationStrategies.BY_SHARING, + typeFullName = typeFullName + ).dynamicTypeHintFullName(typeFullName :: Nil) + // TODO Add dynamicTypeHintFullName to parameterInNode param list + + scope.addToScope(NameConstants.This, thisNode) + + Ast(thisNode) + + private def thisIdentifier(lineNumber: Option[Integer]): NewIdentifier = + val typ = scope.getEnclosingTypeDeclTypeName + newIdentifierNode(NameConstants.This, typ.getOrElse("ANY"), typ.toList, lineNumber) + .code(s"$$${NameConstants.This}") + + private def setParamIndices(asts: Seq[Ast]): Seq[Ast] = + asts.map(_.root).zipWithIndex.foreach { + case (Some(root: NewMethodParameterIn), idx) => + root.index(idx + 1) + + case (root, _) => + logger.debug(s"Trying to set index for unsupported node $root") + } + + asts + + private def composeMethodFullName(methodName: String, isStatic: Boolean): String = + if methodName == NamespaceTraversal.globalNamespaceName then + globalNamespace.fullName + else + val className = getTypeDeclPrefix + val methodDelimiter = + if isStatic then StaticMethodDelimiter else InstanceMethodDelimiter + + val nameWithClass = List(className, Some(methodName)).flatten.mkString(methodDelimiter) + + prependNamespacePrefix(nameWithClass) + + private def astForMethodDecl( + decl: PhpMethodDecl, + bodyPrefixAsts: List[Ast] = Nil, + fullNameOverride: Option[String] = None, + isConstructor: Boolean = false + ): Ast = + val isStatic = decl.modifiers.contains(ModifierTypes.STATIC) + val thisParam = if decl.isClassMethod && !isStatic then + Option(thisParamAstForMethod(decl)) + else + None + + val methodName = decl.name.name + val fullName = fullNameOverride.getOrElse(composeMethodFullName(methodName, isStatic)) + + val signature = s"$UnresolvedSignature(${decl.params.size})" + + val parameters = thisParam.toList ++ decl.params.zipWithIndex.map { case (param, idx) => + astForParam(param, idx + 1) + } + + val constructorModifier = Option.when(isConstructor)(ModifierTypes.CONSTRUCTOR) + val defaultAccessModifier = + Option.unless(containsAccessModifier(decl.modifiers))(ModifierTypes.PUBLIC) + + val allModifiers = constructorModifier ++: defaultAccessModifier ++: decl.modifiers + val modifiers = allModifiers.map(newModifierNode) + val excludedModifiers = Set("MODULE", "LAMBDA") + val modifierString = decl.modifiers.filterNot(excludedModifiers.contains) match + case Nil => "" + case mods => s"${mods.mkString(" ")} " + val methodCode = + s"${modifierString}function $methodName(${parameters.map(_.rootCodeOrEmpty).mkString(",")})" + + val method = methodNode(decl, methodName, methodCode, fullName, Some(signature), filename) + + scope.pushNewScope(method) + + val returnType = decl.returnType.map(_.name).getOrElse(TypeConstants.Any) + + val methodBodyStmts = bodyPrefixAsts ++ decl.stmts.flatMap(astsForStmt) + val methodReturn = newMethodReturnNode(returnType, line = line(decl), column = None) + + val methodBody = blockAst(blockNode(decl), methodBodyStmts) + + scope.popScope() + methodAstWithAnnotations(method, parameters, methodBody, methodReturn, modifiers) + end astForMethodDecl + + private def stmtBodyBlockAst(stmt: PhpStmtWithBody): Ast = + val bodyBlock = blockNode(stmt) + val bodyStmtAsts = stmt.stmts.flatMap(astsForStmt) + Ast(bodyBlock).withChildren(bodyStmtAsts) + + private def astForParam(param: PhpParam, index: Int): Ast = + val evaluationStrategy = + if param.byRef then + EvaluationStrategies.BY_REFERENCE + else + EvaluationStrategies.BY_VALUE + + val typeFullName = param.paramType.map(_.name).getOrElse(TypeConstants.Any) + + val byRefCodePrefix = if param.byRef then "&" else "" + val code = s"$byRefCodePrefix$$${param.name}" + val paramNode = parameterInNode( + param, + param.name, + code, + index, + param.isVariadic, + evaluationStrategy, + typeFullName + ) + + scope.addToScope(param.name, paramNode) + + Ast(paramNode) + end astForParam + + private def astForExpr(expr: PhpExpr): Ast = + expr match + case funcCallExpr: PhpCallExpr => astForCall(funcCallExpr) + case variableExpr: PhpVariable => astForVariableExpr(variableExpr) + case nameExpr: PhpNameExpr => astForNameExpr(nameExpr) + case assignExpr: PhpAssignment => astForAssignment(assignExpr) + case scalarExpr: PhpScalar => astForScalar(scalarExpr) + case binaryOp: PhpBinaryOp => astForBinOp(binaryOp) + case unaryOp: PhpUnaryOp => astForUnaryOp(unaryOp) + case castExpr: PhpCast => astForCastExpr(castExpr) + case isSetExpr: PhpIsset => astForIsSetExpr(isSetExpr) + case printExpr: PhpPrint => astForPrintExpr(printExpr) + case ternaryOp: PhpTernaryOp => astForTernaryOp(ternaryOp) + case throwExpr: PhpThrowExpr => astForThrow(throwExpr) + case cloneExpr: PhpCloneExpr => astForClone(cloneExpr) + case emptyExpr: PhpEmptyExpr => astForEmpty(emptyExpr) + case evalExpr: PhpEvalExpr => astForEval(evalExpr) + case exitExpr: PhpExitExpr => astForExit(exitExpr) + case arrayExpr: PhpArrayExpr => astForArrayExpr(arrayExpr) + case listExpr: PhpListExpr => astForListExpr(listExpr) + case newExpr: PhpNewExpr => astForNewExpr(newExpr) + case matchExpr: PhpMatchExpr => astForMatchExpr(matchExpr) + case yieldExpr: PhpYieldExpr => astForYieldExpr(yieldExpr) + case closure: PhpClosureExpr => astForClosureExpr(closure) + case yieldFromExpr: PhpYieldFromExpr => astForYieldFromExpr(yieldFromExpr) + case classConstFetchExpr: PhpClassConstFetchExpr => + astForClassConstFetchExpr(classConstFetchExpr) + case constFetchExpr: PhpConstFetchExpr => astForConstFetchExpr(constFetchExpr) + case arrayDimFetchExpr: PhpArrayDimFetchExpr => + astForArrayDimFetchExpr(arrayDimFetchExpr) + case errorSuppressExpr: PhpErrorSuppressExpr => + astForErrorSuppressExpr(errorSuppressExpr) + case instanceOfExpr: PhpInstanceOfExpr => astForInstanceOfExpr(instanceOfExpr) + case propertyFetchExpr: PhpPropertyFetchExpr => + astForPropertyFetchExpr(propertyFetchExpr) + case includeExpr: PhpIncludeExpr => astForIncludeExpr(includeExpr) + case shellExecExpr: PhpShellExecExpr => astForShellExecExpr(shellExecExpr) + case null => + logger.debug("expr was null") + ??? + case other => throw new NotImplementedError( + s"unexpected expression '$other' of type ${other.getClass}" + ) + + private def intToLiteralAst(num: Int): Ast = + Ast(NewLiteral().code(num.toString).typeFullName(TypeConstants.Int)) + + private def astForBreakStmt(breakStmt: PhpBreakStmt): Ast = + val code = breakStmt.num.map(num => s"break($num)").getOrElse("break") + val breakNode = controlStructureNode(breakStmt, ControlStructureTypes.BREAK, code) + + val argument = breakStmt.num.map(intToLiteralAst) + + controlStructureAst(breakNode, None, argument.toList) + + private def astForContinueStmt(continueStmt: PhpContinueStmt): Ast = + val code = continueStmt.num.map(num => s"continue($num)").getOrElse("continue") + val continueNode = controlStructureNode(continueStmt, ControlStructureTypes.CONTINUE, code) + + val argument = continueStmt.num.map(intToLiteralAst) + + controlStructureAst(continueNode, None, argument.toList) + + private def astForWhileStmt(whileStmt: PhpWhileStmt): Ast = + val condition = astForExpr(whileStmt.cond) + val lineNumber = line(whileStmt) + val code = s"while (${condition.rootCodeOrEmpty})" + val body = stmtBodyBlockAst(whileStmt) + + whileAst(Option(condition), List(body), Option(code), lineNumber) + + private def astForDoStmt(doStmt: PhpDoStmt): Ast = + val condition = astForExpr(doStmt.cond) + val lineNumber = line(doStmt) + val code = s"do {...} while (${condition.rootCodeOrEmpty})" + val body = stmtBodyBlockAst(doStmt) + + doWhileAst(Option(condition), List(body), Option(code), lineNumber) + + private def astForForStmt(stmt: PhpForStmt): Ast = + val lineNumber = line(stmt) + + val initAsts = stmt.inits.map(astForExpr) + val conditionAsts = stmt.conditions.map(astForExpr) + val loopExprAsts = stmt.loopExprs.map(astForExpr) + + val bodyAst = stmtBodyBlockAst(stmt) + + val initCode = initAsts.map(_.rootCodeOrEmpty).mkString(",") + val conditionCode = conditionAsts.map(_.rootCodeOrEmpty).mkString(",") + val loopExprCode = loopExprAsts.map(_.rootCodeOrEmpty).mkString(",") + val forCode = s"for ($initCode;$conditionCode;$loopExprCode)" + + val forNode = controlStructureNode(stmt, ControlStructureTypes.FOR, forCode) + forAst(forNode, Nil, initAsts, conditionAsts, loopExprAsts, bodyAst) + + private def astForIfStmt(ifStmt: PhpIfStmt): Ast = + val condition = astForExpr(ifStmt.cond) + + val thenAst = stmtBodyBlockAst(ifStmt) + + val elseAst = ifStmt.elseIfs match + case Nil => ifStmt.elseStmt.map(els => stmtBodyBlockAst(els)).toList + + case elseIf :: rest => + val newIfStmt = + PhpIfStmt(elseIf.cond, elseIf.stmts, rest, ifStmt.elseStmt, elseIf.attributes) + val wrappingBlock = blockNode(elseIf) + val wrappedAst = Ast(wrappingBlock).withChild(astForIfStmt(newIfStmt)) :: Nil + wrappedAst + + val conditionCode = condition.rootCodeOrEmpty + val ifNode = controlStructureNode(ifStmt, ControlStructureTypes.IF, s"if ($conditionCode)") + + controlStructureAst(ifNode, Option(condition), thenAst :: elseAst) + end astForIfStmt + + private def astForSwitchStmt(stmt: PhpSwitchStmt): Ast = + val conditionAst = astForExpr(stmt.condition) + + val switchNode = + controlStructureNode( + stmt, + ControlStructureTypes.SWITCH, + s"switch (${conditionAst.rootCodeOrEmpty})" + ) + + val switchBodyBlock = blockNode(stmt) + val entryAsts = stmt.cases.flatMap(astsForSwitchCase) + val switchBody = Ast(switchBodyBlock).withChildren(entryAsts) + + controlStructureAst(switchNode, Option(conditionAst), switchBody :: Nil) + + private def astForTryStmt(stmt: PhpTryStmt): Ast = + val tryBody = stmtBodyBlockAst(stmt) + val catches = stmt.catches.map(astForCatchStmt) + val finallyBody = stmt.finallyStmt.map(fin => stmtBodyBlockAst(fin)) + + val tryNode = controlStructureNode(stmt, ControlStructureTypes.TRY, "try { ... }") + + tryCatchAst(tryNode, tryBody, catches, finallyBody) + + private def astForReturnStmt(stmt: PhpReturnStmt): Ast = + val maybeExprAst = stmt.expr.map(astForExpr) + val code = s"return ${maybeExprAst.map(_.rootCodeOrEmpty).getOrElse("")}" + + val node = returnNode(stmt, code) + + returnAst(node, maybeExprAst.toList) + + private def astForClassLikeStmt(stmt: PhpClassLikeStmt): Ast = + stmt.name match + case None => astForAnonymousClass(stmt) + case Some(name) => astForNamedClass(stmt, name) + + private def astForGotoStmt(stmt: PhpGotoStmt): Ast = + val label = stmt.label.name + val code = s"goto $label" + + val gotoNode = controlStructureNode(stmt, ControlStructureTypes.GOTO, code) + + val jumpLabel = NewJumpLabel() + .name(label) + .code(label) + .lineNumber(line(stmt)) + + controlStructureAst(gotoNode, condition = None, children = Ast(jumpLabel) :: Nil) + + private def astForLabelStmt(stmt: PhpLabelStmt): Ast = + val label = stmt.label.name + + val jumpTarget = NewJumpTarget() + .name(label) + .code(label) + .lineNumber(line(stmt)) + + Ast(jumpTarget) + + private def astForNamespaceStmt(stmt: PhpNamespaceStmt): Ast = + val name = stmt.name.map(_.name).getOrElse(NameConstants.Unknown) + val fullName = s"$filename:$name" + + val namespaceBlock = NewNamespaceBlock() + .name(name) + .fullName(fullName) + + scope.pushNewScope(namespaceBlock) + val bodyStmts = astsForClassLikeBody(stmt, stmt.stmts, createDefaultConstructor = false) + scope.popScope() + + Ast(namespaceBlock).withChildren(bodyStmts) + + private def astForDeclareStmt(stmt: PhpDeclareStmt): Ast = + val declareAssignAsts = stmt.declares.map(astForDeclareItem) + val declareCode = + s"${PhpOperators.declareFunc}(${declareAssignAsts.map(_.rootCodeOrEmpty).mkString(",")})" + val declareNode = + newOperatorCallNode(PhpOperators.declareFunc, declareCode, line = line(stmt)) + val declareAst = callAst(declareNode, declareAssignAsts) + + stmt.stmts match + case Some(stmtList) => + val stmtAsts = stmtList.flatMap(astsForStmt) + Ast(blockNode(stmt)) + .withChild(declareAst) + .withChildren(stmtAsts) + + case None => declareAst + + private def astForDeclareItem(item: PhpDeclareItem): Ast = + val key = identifierNode(item, item.key.name, item.key.name, "ANY") + val value = astForExpr(item.value) + val code = s"${key.name}=${value.rootCodeOrEmpty}" + + val declareAssignment = newOperatorCallNode(Operators.assignment, code, line = line(item)) + callAst(declareAssignment, Ast(key) :: value :: Nil) + + private def astForHaltCompilerStmt(stmt: PhpHaltCompilerStmt): Ast = + val call = newOperatorCallNode( + NameConstants.HaltCompiler, + s"${NameConstants.HaltCompiler}()", + Some(TypeConstants.Void), + line(stmt), + column(stmt) + ) + + Ast(call) + + private def astForUnsetStmt(stmt: PhpUnsetStmt): Ast = + val name = PhpOperators.unset + val args = stmt.vars.map(astForExpr) + val code = s"$name(${args.map(_.rootCodeOrEmpty).mkString(", ")})" + val callNode = newOperatorCallNode( + name, + code, + typeFullName = Some(TypeConstants.Void), + line = line(stmt) + ) + .methodFullName(PhpOperators.unset) + callAst(callNode, args) + + private def astForGlobalStmt(stmt: PhpGlobalStmt): Ast = + // This isn't an accurater representation of what `global` does, but with things like `global $$x` being possible, + // it's very difficult to figure out correct scopes for global variables. + + val varsAsts = stmt.vars.map(astForExpr) + val code = s"${PhpOperators.global} ${varsAsts.map(_.rootCodeOrEmpty).mkString(", ")}" + + val globalCallNode = + newOperatorCallNode(PhpOperators.global, code, Some(TypeConstants.Void), line(stmt)) + + callAst(globalCallNode, varsAsts) + + private def astForUseStmt(stmt: PhpUseStmt): Ast = + // TODO Use useType + scope to get better name info + val imports = stmt.uses.map(astForUseUse(_)) + wrapMultipleInBlock(imports, line(stmt)) + + private def astForGroupUseStmt(stmt: PhpGroupUseStmt): Ast = + // TODO Use useType + scope to get better name info + val groupPrefix = s"${stmt.prefix.name}\\" + val imports = stmt.uses.map(astForUseUse(_, groupPrefix)) + wrapMultipleInBlock(imports, line(stmt)) + + private def astForKeyValPair(key: PhpExpr, value: PhpExpr, lineNo: Option[Integer]): Ast = + val keyAst = astForExpr(key) + val valueAst = astForExpr(value) + + val code = s"${keyAst.rootCodeOrEmpty} => ${valueAst.rootCodeOrEmpty}" + val callNode = newOperatorCallNode(PhpOperators.doubleArrow, code, line = lineNo) + callAst(callNode, keyAst :: valueAst :: Nil) + + private def astForForeachStmt(stmt: PhpForeachStmt): Ast = + val iteratorAst = astForExpr(stmt.iterExpr) + val iterIdentifier = getTmpIdentifier(stmt, maybeTypeFullName = None, prefix = "iter_") + + val assignItemTargetAst = stmt.keyVar match + case Some(key) => astForKeyValPair(key, stmt.valueVar, line(stmt)) + case None => astForExpr(stmt.valueVar) + + // Initializer asts + // - Iterator assign + val iterValue = astForExpr(stmt.iterExpr) + val iteratorAssignAst = simpleAssignAst(Ast(iterIdentifier), iterValue, line(stmt)) + + // - Assigned item assign + val itemInitAst = getItemAssignAstForForeach(stmt, assignItemTargetAst, iterIdentifier.copy) + + // Condition ast + val isNullName = PhpOperators.isNull + val valueAst = astForExpr(stmt.valueVar) + val isNullCode = s"$isNullName(${valueAst.rootCodeOrEmpty})" + val isNullCall = + newOperatorCallNode(isNullName, isNullCode, Some(TypeConstants.Bool), line(stmt)) + .methodFullName(PhpOperators.isNull) + val notIsNull = + newOperatorCallNode(Operators.logicalNot, s"!$isNullCode", line = line(stmt)) + val isNullAst = callAst(isNullCall, valueAst :: Nil) + val conditionAst = callAst(notIsNull, isNullAst :: Nil) + + // Update asts + val nextIterIdent = Ast(iterIdentifier.copy) + val nextSignature = "void()" + val nextCallCode = s"${nextIterIdent.rootCodeOrEmpty}->next()" + val nextCallNode = callNode( + stmt, + nextCallCode, + "next", + "Iterator.next", + DispatchTypes.DYNAMIC_DISPATCH, + Some(nextSignature), + Some(TypeConstants.Any) + ) + val nextCallAst = callAst(nextCallNode, base = Option(nextIterIdent)) + val itemUpdateAst = itemInitAst.root match + case Some(initRoot: AstNodeNew) => itemInitAst.subTreeCopy(initRoot) + case _ => + logger.debug(s"Could not copy foreach init ast in $filename") + Ast() + + val bodyAst = stmtBodyBlockAst(stmt) + + val ampPrefix = if stmt.assignByRef then "&" else "" + val foreachCode = + s"foreach (${iteratorAst.rootCodeOrEmpty} as $ampPrefix${assignItemTargetAst.rootCodeOrEmpty})" + val foreachNode = controlStructureNode(stmt, ControlStructureTypes.FOR, foreachCode) + Ast(foreachNode) + .withChild(wrapMultipleInBlock(iteratorAssignAst :: itemInitAst :: Nil, line(stmt))) + .withChild(conditionAst) + .withChild(wrapMultipleInBlock(nextCallAst :: itemUpdateAst :: Nil, line(stmt))) + .withChild(bodyAst) + .withConditionEdges(foreachNode, conditionAst.root.toList) + end astForForeachStmt + + private def getItemAssignAstForForeach( + stmt: PhpForeachStmt, + assignItemTargetAst: Ast, + iteratorIdentifier: NewIdentifier + ): Ast = + val iteratorIdentifierAst = Ast(iteratorIdentifier) + val currentCallSignature = s"$UnresolvedSignature(0)" + val currentCallCode = s"${iteratorIdentifierAst.rootCodeOrEmpty}->current()" + val currentCallNode = callNode( + stmt, + currentCallCode, + "current", + "Iterator.current", + DispatchTypes.DYNAMIC_DISPATCH, + Some(currentCallSignature), + Some(TypeConstants.Any) + ); + val currentCallAst = callAst(currentCallNode, base = Option(iteratorIdentifierAst)) + + val valueAst = if stmt.assignByRef then + val addressOfCode = s"&${currentCallAst.rootCodeOrEmpty}" + val addressOfCall = + newOperatorCallNode(Operators.addressOf, addressOfCode, line = line(stmt)) + callAst(addressOfCall, currentCallAst :: Nil) + else + currentCallAst + + simpleAssignAst(assignItemTargetAst, valueAst, line(stmt)) + end getItemAssignAstForForeach + + private def simpleAssignAst(target: Ast, source: Ast, lineNo: Option[Integer]): Ast = + val code = s"${target.rootCodeOrEmpty} = ${source.rootCodeOrEmpty}" + val callNode = newOperatorCallNode(Operators.assignment, code, line = lineNo) + callAst(callNode, target :: source :: Nil) + + private def astforTraitUseStmt(stmt: PhpTraitUseStmt): Ast = + // TODO Actually implement this + Ast() + + private def astForUseUse(stmt: PhpUseUse, namePrefix: String = ""): Ast = + val originalName = s"$namePrefix${stmt.originalName.name}" + val aliasCode = stmt.alias.map(alias => s" as ${alias.name}").getOrElse("") + val typeCode = stmt.useType match + case PhpUseType.Function => s"function " + case PhpUseType.Constant => s"const " + case _ => "" + val code = s"use $typeCode$originalName$aliasCode" + + val importNode = NewImport() + .importedEntity(originalName) + .importedAs(stmt.alias.map(_.name)) + .isExplicit(true) + .code(code) + + Ast(importNode) + + private def astsForStaticStmt(stmt: PhpStaticStmt): List[Ast] = + stmt.vars.flatMap { staticVarDecl => + val variableAst = astForVariableExpr(staticVarDecl.variable) + val maybeValueAst = staticVarDecl.defaultValue.map(astForExpr) + + val code = variableAst.rootCode.getOrElse(NameConstants.Unknown) + val name = variableAst.root match + case Some(identifier: NewIdentifier) => identifier.name + case _ => code + + val local = localNode( + stmt, + name, + s"static $code", + variableAst.rootType.getOrElse(TypeConstants.Any) + ) + scope.addToScope(local.name, local) + + variableAst.root.collect { case identifier: NewIdentifier => + diffGraph.addEdge(identifier, local, EdgeTypes.REF) + } + + val defaultAssignAst = maybeValueAst.map { valueAst => + val valueCode = s"static $code = ${valueAst.rootCodeOrEmpty}" + val assignNode = + newOperatorCallNode(Operators.assignment, valueCode, line = line(stmt)) + callAst(assignNode, variableAst :: valueAst :: Nil) + } + + Ast(local) :: defaultAssignAst.toList + } + + private def astForAnonymousClass(stmt: PhpClassLikeStmt): Ast = + // TODO + Ast() + + def codeForClassStmt(stmt: PhpClassLikeStmt, name: PhpNameExpr): String = + // TODO Extend for anonymous classes + val extendsString = stmt.extendsNames match + case Nil => "" + case names => s" extends ${names.map(_.name).mkString(", ")}" + val implementsString = + if stmt.implementedInterfaces.isEmpty then + "" + else + s" implements ${stmt.implementedInterfaces.map(_.name).mkString(", ")}" + + s"${stmt.classLikeType} ${name.name}$extendsString$implementsString" + + private def astForNamedClass(stmt: PhpClassLikeStmt, name: PhpNameExpr): Ast = + val inheritsFrom = (stmt.extendsNames ++ stmt.implementedInterfaces).map(_.name) + val code = codeForClassStmt(stmt, name) + + val fullName = + if name.name == NamespaceTraversal.globalNamespaceName then + globalNamespace.fullName + else + prependNamespacePrefix(name.name) + + val typeDecl = + typeDeclNode(stmt, name.name, fullName, filename, code, inherits = inheritsFrom) + + val createDefaultConstructor = stmt.hasConstructor + + scope.pushNewScope(typeDecl) + val bodyStmts = astsForClassLikeBody(stmt, stmt.stmts, createDefaultConstructor) + val modifiers = stmt.modifiers.map(newModifierNode).map(Ast(_)) + scope.popScope() + + Ast(typeDecl).withChildren(modifiers).withChildren(bodyStmts) + end astForNamedClass + + private def astForStaticAndConstInits: Option[Ast] = + scope.getConstAndStaticInits match + case Nil => None + + case inits => + val signature = s"${TypeConstants.Void}()" + val fullName = composeMethodFullName(StaticInitMethodName, isStatic = true) + val ast = staticInitMethodAst( + inits, + fullName, + Option(signature), + TypeConstants.Void, + fileName = Some(filename) + ) + Option(ast) + + private def astsForClassLikeBody( + classLike: PhpStmt, + bodyStmts: List[PhpStmt], + createDefaultConstructor: Boolean + ): List[Ast] = + val classConsts = + bodyStmts.collect { case cs: PhpConstStmt => cs }.flatMap(astsForConstStmt) + val properties = + bodyStmts.collect { case cp: PhpPropertyStmt => cp }.flatMap(astsForPropertyStmt) + + val explicitConstructorAst = bodyStmts.collectFirst { + case m: PhpMethodDecl if m.name.name == ConstructorMethodName => astForConstructor(m) + } + + val constructorAst = + explicitConstructorAst.orElse( + Option.when(createDefaultConstructor)(defaultConstructorAst(classLike)) + ) + + val otherBodyStmts = bodyStmts.flatMap { + case _: PhpConstStmt => Nil // Handled above + + case _: PhpPropertyStmt => Nil // Handled above + + case method: PhpMethodDecl if method.name.name == ConstructorMethodName => + Nil // Handled above + + // Not all statements are supported in class bodies, but since this is re-used for namespaces + // we allow that here. + case stmt => astsForStmt(stmt) + } + + val clinitAst = astForStaticAndConstInits + val anonymousMethodAsts = scope.getAndClearAnonymousMethods + + List( + classConsts, + properties, + clinitAst, + constructorAst, + anonymousMethodAsts, + otherBodyStmts + ).flatten + end astsForClassLikeBody + + private def astForConstructor(constructorDecl: PhpMethodDecl): Ast = + val fieldInits = scope.getFieldInits + astForMethodDecl(constructorDecl, fieldInits, isConstructor = true) + + private def prependNamespacePrefix(name: String): String = + scope.getEnclosingNamespaceNames.filterNot( + _ == NamespaceTraversal.globalNamespaceName + ) match + case Nil => name + case names => names.appended(name).mkString(NamespaceDelimiter) + + private def getTypeDeclPrefix: Option[String] = + scope.getEnclosingTypeDeclTypeName + .filterNot(_ == NamespaceTraversal.globalNamespaceName) + + private def defaultConstructorAst(originNode: PhpNode): Ast = + val fullName = composeMethodFullName(ConstructorMethodName, isStatic = false) + + val signature = s"$UnresolvedSignature(0)" + + val modifiers = List( + ModifierTypes.VIRTUAL, + ModifierTypes.PUBLIC, + ModifierTypes.CONSTRUCTOR + ).map(newModifierNode) + + val thisParam = thisParamAstForMethod(originNode) + + val method = methodNode( + originNode, + ConstructorMethodName, + fullName, + fullName, + Some(signature), + filename + ) + + val methodBody = blockAst(blockNode(originNode), scope.getFieldInits) + + val methodReturn = newMethodReturnNode(TypeConstants.Any, line = None, column = None) + + methodAstWithAnnotations(method, thisParam :: Nil, methodBody, methodReturn, modifiers) + end defaultConstructorAst + + private def astForMemberAssignment( + memberNode: NewMember, + valueExpr: PhpExpr, + isField: Boolean + ): Ast = + val targetAst = if isField then + val code = s"$$this->${memberNode.name}" + val fieldAccessNode = + newOperatorCallNode(Operators.fieldAccess, code, line = memberNode.lineNumber) + val identifier = thisIdentifier(memberNode.lineNumber) + val thisParam = scope.lookupVariable(NameConstants.This) + val fieldIdentifier = newFieldIdentifierNode(memberNode.name, memberNode.lineNumber) + callAst(fieldAccessNode, List(identifier, fieldIdentifier).map(Ast(_))).withRefEdges( + identifier, + thisParam.toList + ) + else + val identifierCode = memberNode.code.replaceAll("const ", "").replaceAll("case ", "") + val typeFullName = Option(memberNode.typeFullName) + val identifier = newIdentifierNode(memberNode.name, typeFullName.getOrElse("ANY")) + .code(identifierCode) + Ast(identifier).withRefEdge(identifier, memberNode) + val value = astForExpr(valueExpr) + + val assignmentCode = s"${targetAst.rootCodeOrEmpty} = ${value.rootCodeOrEmpty}" + val callNode = + newOperatorCallNode(Operators.assignment, assignmentCode, line = memberNode.lineNumber) + + callAst(callNode, List(targetAst, value)) + end astForMemberAssignment + + private def astsForConstStmt(stmt: PhpConstStmt): List[Ast] = + stmt.consts.map { constDecl => + val finalModifier = Ast(newModifierNode(ModifierTypes.FINAL)) + // `final const` is not allowed, so this is a safe way to represent constants in the CPG + val modifierAsts = finalModifier :: stmt.modifiers.map(newModifierNode).map(Ast(_)) + + val name = constDecl.name.name + val code = s"const $name" + val someValue = Option(constDecl.value) + astForConstOrFieldValue( + stmt, + name, + code, + someValue, + scope.addConstOrStaticInitToScope, + isField = false + ) + .withChildren(modifierAsts) + } + + private def astForEnumCase(stmt: PhpEnumCaseStmt): Ast = + val finalModifier = Ast(newModifierNode(ModifierTypes.FINAL)) + + val name = stmt.name.name + val code = s"case $name" + + astForConstOrFieldValue( + stmt, + name, + code, + stmt.expr, + scope.addConstOrStaticInitToScope, + isField = false + ) + .withChild(finalModifier) + + private def astsForPropertyStmt(stmt: PhpPropertyStmt): List[Ast] = + stmt.variables.map { varDecl => + val modifierAsts = stmt.modifiers.map(newModifierNode).map(Ast(_)) + + val name = varDecl.name.name + astForConstOrFieldValue( + stmt, + name, + s"$$$name", + varDecl.defaultValue, + scope.addFieldInitToScope, + isField = true + ) + .withChildren(modifierAsts) + } + + private def astForConstOrFieldValue( + originNode: PhpNode, + name: String, + code: String, + value: Option[PhpExpr], + addToScope: Ast => Unit, + isField: Boolean + ): Ast = + val member = memberNode(originNode, name, code, TypeConstants.Any) + + value match + case Some(v) => + val assignAst = astForMemberAssignment(member, v, isField) + addToScope(assignAst) + case None => // Nothing to do here + Ast(member) + + private def astForCatchStmt(stmt: PhpCatchStmt): Ast = + // TODO Add variable at some point. Current implementation is consistent with C++. + stmtBodyBlockAst(stmt) + + private def astsForSwitchCase(caseStmt: PhpCaseStmt): List[Ast] = + val maybeConditionAst = caseStmt.condition.map(astForExpr) + val jumpTarget = maybeConditionAst match + case Some(conditionAst) => + NewJumpTarget().name("case").code(s"case ${conditionAst.rootCodeOrEmpty}") + case None => NewJumpTarget().name("default").code("default") + jumpTarget.lineNumber(line(caseStmt)) + + val stmtAsts = caseStmt.stmts.flatMap(astsForStmt) + + Ast(jumpTarget) :: stmtAsts + + private def codeForMethodCall(call: PhpCallExpr, targetAst: Ast, name: String): String = + val callOperator = if call.isNullSafe then "?->" else "->" + s"${targetAst.rootCodeOrEmpty}$callOperator$name" + + private def codeForStaticMethodCall(call: PhpCallExpr, name: String): String = + val className = + call.target + .map(astForExpr) + .map(_.rootCode.getOrElse(UnresolvedNamespace)) + .getOrElse(UnresolvedNamespace) + s"$className::$name" + + private def astForCall(call: PhpCallExpr): Ast = + val arguments = call.args.map(astForCallArg) + + val targetAst = Option.unless(call.isStatic)(call.target.map(astForExpr)).flatten + + val nameAst = + Option.unless(call.methodName.isInstanceOf[PhpNameExpr])(astForExpr(call.methodName)) + val name = + nameAst + .map(_.rootCodeOrEmpty) + .getOrElse(call.methodName match + case nameExpr: PhpNameExpr => nameExpr.name + case other => + logger.debug( + s"Found unexpected call target type: Crash for now to handle properly later: $other" + ) + ??? + ) + + val argsCode = arguments + .zip(call.args.collect { case x: PhpArg => x.unpack }) + .map { + case (arg, true) => s"...${arg.rootCodeOrEmpty}" + case (arg, false) => arg.rootCodeOrEmpty + } + .mkString(",") + + val codePrefix = + if !call.isStatic && targetAst.isDefined then + codeForMethodCall(call, targetAst.get, name) + else if call.isStatic then + codeForStaticMethodCall(call, name) + else + name + + val code = s"$codePrefix($argsCode)" + + val dispatchType = + if call.isStatic || call.target.isEmpty then + DispatchTypes.STATIC_DISPATCH + else + DispatchTypes.DYNAMIC_DISPATCH + + val fullName = call.target match + // Static method call with a known class name + case Some(nameExpr: PhpNameExpr) if call.isStatic => + if nameExpr.name == "self" then composeMethodFullName(name, call.isStatic) + else s"${nameExpr.name}${StaticMethodDelimiter}$name" + + case Some(expr) => + s"$UnresolvedNamespace\\$codePrefix" + + case None if PhpBuiltins.FuncNames.contains(name) => + // No signature/namespace for MFN for builtin functions to ensure stable names as type info improves. + name + + // Function call + case None => + composeMethodFullName(name, call.isStatic) + + // Use method signature for methods that can be linked to avoid varargs issue. + val signature = s"$UnresolvedSignature(${call.args.size})" + val callRoot = callNode( + call, + code, + name, + fullName, + dispatchType, + Some(signature), + Some(TypeConstants.Any) + ) + + val receiverAst = (targetAst, nameAst) match + case (Some(target), Some(n)) => + val fieldAccess = + newOperatorCallNode(Operators.fieldAccess, codePrefix, line = line(call)) + Option(callAst(fieldAccess, target :: n :: Nil)) + case (Some(target), None) => Option(target) + case (None, Some(n)) => Option(n) + case (None, None) => None + + callAst(callRoot, arguments, base = receiverAst) + end astForCall + + private def astForCallArg(arg: PhpArgument): Ast = + arg match + case PhpArg(expr, _, _, _, _) => + astForExpr(expr) + + case _: PhpVariadicPlaceholder => + val identifier = + identifierNode(arg, "...", "...", TypeConstants.VariadicPlaceholder) + Ast(identifier) + + private def astForVariableExpr(variable: PhpVariable): Ast = + // TODO Need to figure out variable variables. Maybe represent as some kind of call? + val valueAst = astForExpr(variable.value) + + valueAst.root.collect { case root: ExpressionNew => + root.code = s"$$${root.code}" + } + + valueAst.root.collect { case root: NewIdentifier => + root.lineNumber = line(variable) + } + + valueAst + + private def astForNameExpr(expr: PhpNameExpr): Ast = + val identifier = identifierNode(expr, expr.name, expr.name, TypeConstants.Any) + + scope.lookupVariable(identifier.name).foreach { declaringNode => + diffGraph.addEdge(identifier, declaringNode, EdgeTypes.REF) + } + + Ast(identifier) + + /** This is used to rewrite the short form $xs[] = as array_push($xs, ) + * to avoid having to handle the empty array access operator as a special case in the dataflow + * engine. + * + * This representation is technically wrong in the case where the shorthand is used to + * initialise a new array (since PHP expects the first argument to array_push to be an existing + * array). This shouldn't affect dataflow, however. + */ + private def astForEmptyArrayDimAssign( + assignment: PhpAssignment, + arrayDimFetch: PhpArrayDimFetchExpr + ): Ast = + val attrs = assignment.attributes + val arrayPushArgs = List(arrayDimFetch.variable, assignment.source).map(PhpArg(_)) + val arrayPushCall = PhpCallExpr( + target = None, + methodName = PhpNameExpr("array_push", attrs), + args = arrayPushArgs, + isNullSafe = false, + isStatic = true, + attributes = attrs + ) + val arrayPushAst = astForCall(arrayPushCall) + arrayPushAst.root.collect { case astRoot: NewCall => + val args = + arrayPushAst.argEdges + .filter(_.src == astRoot) + .map(_.dst) + .collect { case arg: ExpressionNew => arg } + .sortBy(_.argumentIndex) + + if args.size != 2 then + val position = s"${line(assignment).getOrElse("")}:${filename}" + logger.debug( + s"Expected 2 call args for emptyArrayDimAssign. Not resetting code: ${position}" + ) + else + val codeOverride = s"${args.head.code}[] = ${args.last.code}" + astRoot.code(codeOverride) + } + arrayPushAst + end astForEmptyArrayDimAssign + + private def astForAssignment(assignment: PhpAssignment): Ast = + assignment.target match + case arrayDimFetch: PhpArrayDimFetchExpr if arrayDimFetch.dimension.isEmpty => + // Rewrite `$xs[] = ` as `array_push($xs, )` to simplify finding dataflows. + astForEmptyArrayDimAssign(assignment, arrayDimFetch) + + case _ => + val operatorName = assignment.assignOp + + val targetAst = astForExpr(assignment.target) + val sourceAst = astForExpr(assignment.source) + + // TODO Handle ref assigns properly (if needed). + val refSymbol = if assignment.isRefAssign then "&" else "" + val symbol = operatorSymbols.getOrElse(assignment.assignOp, assignment.assignOp) + val code = + s"${targetAst.rootCodeOrEmpty} $symbol $refSymbol${sourceAst.rootCodeOrEmpty}" + + val callNode = newOperatorCallNode(operatorName, code, line = line(assignment)) + callAst(callNode, List(targetAst, sourceAst)) + + private def astForEncapsed(encapsed: PhpEncapsed): Ast = + val args = encapsed.parts.map(astForExpr) + val code = args.map(_.rootCodeOrEmpty).mkString(" . ") + + args match + case singleArg :: Nil => singleArg + case _ => + val callNode = newOperatorCallNode( + PhpOperators.encaps, + code, + Some(TypeConstants.String), + line(encapsed) + ) + callAst(callNode, args) + + private def astForScalar(scalar: PhpScalar): Ast = + scalar match + case encapsed: PhpEncapsed => astForEncapsed(encapsed) + case simpleScalar: PhpSimpleScalar => + Ast(literalNode(scalar, simpleScalar.value, simpleScalar.typeFullName)) + case null => + logger.debug("scalar was null") + ??? + + private def astForBinOp(binOp: PhpBinaryOp): Ast = + val leftAst = astForExpr(binOp.left) + val rightAst = astForExpr(binOp.right) + + val symbol = operatorSymbols.getOrElse(binOp.operator, binOp.operator) + val code = s"${leftAst.rootCodeOrEmpty} $symbol ${rightAst.rootCodeOrEmpty}" + + val callNode = newOperatorCallNode(binOp.operator, code, line = line(binOp)) + + callAst(callNode, List(leftAst, rightAst)) + + private def isPostfixOperator(operator: String): Boolean = + Set(Operators.postDecrement, Operators.postIncrement).contains(operator) + + private def astForUnaryOp(unaryOp: PhpUnaryOp): Ast = + val exprAst = astForExpr(unaryOp.expr) + + val symbol = operatorSymbols.getOrElse(unaryOp.operator, unaryOp.operator) + val code = + if isPostfixOperator(unaryOp.operator) then + s"${exprAst.rootCodeOrEmpty}$symbol" + else + s"$symbol${exprAst.rootCodeOrEmpty}" + + val callNode = newOperatorCallNode(unaryOp.operator, code, line = line(unaryOp)) + + callAst(callNode, exprAst :: Nil) + + private def astForCastExpr(castExpr: PhpCast): Ast = + val typeFullName = castExpr.typ + val typ = typeRefNode(castExpr, typeFullName, typeFullName) + + val expr = astForExpr(castExpr.expr) + val codeStr = s"($typeFullName) ${expr.rootCodeOrEmpty}" + + val callNode = + newOperatorCallNode(name = Operators.cast, codeStr, Some(typeFullName), line(castExpr)) + + callAst(callNode, Ast(typ) :: expr :: Nil) + + private def astForIsSetExpr(isSetExpr: PhpIsset): Ast = + val name = PhpOperators.issetFunc + val args = isSetExpr.vars.map(astForExpr) + val code = s"$name(${args.map(_.rootCodeOrEmpty).mkString(",")})" + + val callNode = + newOperatorCallNode( + name, + code, + typeFullName = Some(TypeConstants.Bool), + line = line(isSetExpr) + ) + .methodFullName(PhpOperators.issetFunc) + + callAst(callNode, args) + private def astForPrintExpr(printExpr: PhpPrint): Ast = + val name = PhpOperators.printFunc + val arg = astForExpr(printExpr.expr) + val code = s"$name(${arg.rootCodeOrEmpty})" + + val callNode = + newOperatorCallNode( + name, + code, + typeFullName = Some(TypeConstants.Int), + line = line(printExpr) + ) + .methodFullName(PhpOperators.printFunc) + + callAst(callNode, arg :: Nil) + + private def astForTernaryOp(ternaryOp: PhpTernaryOp): Ast = + val conditionAst = astForExpr(ternaryOp.condition) + val maybeThenAst = ternaryOp.thenExpr.map(astForExpr) + val elseAst = astForExpr(ternaryOp.elseExpr) + + val operatorName = + if maybeThenAst.isDefined then Operators.conditional else PhpOperators.elvisOp + val code = maybeThenAst match + case Some(thenAst) => + s"${conditionAst.rootCodeOrEmpty} ? ${thenAst.rootCodeOrEmpty} : ${elseAst.rootCodeOrEmpty}" + case None => s"${conditionAst.rootCodeOrEmpty} ?: ${elseAst.rootCodeOrEmpty}" + + val callNode = newOperatorCallNode(operatorName, code, line = line(ternaryOp)) + + val args = List(Option(conditionAst), maybeThenAst, Option(elseAst)).flatten + callAst(callNode, args) + + private def astForThrow(expr: PhpThrowExpr): Ast = + val thrownExpr = astForExpr(expr.expr) + val code = s"throw ${thrownExpr.rootCodeOrEmpty}" + + val throwNode = controlStructureNode(expr, ControlStructureTypes.THROW, code) + + Ast(throwNode).withChild(thrownExpr) + + private def astForClone(expr: PhpCloneExpr): Ast = + val name = PhpOperators.cloneFunc + val argAst = astForExpr(expr.expr) + val argType = argAst.rootType.orElse(Some(TypeConstants.Any)) + val code = s"$name ${argAst.rootCodeOrEmpty}" + + val callNode = newOperatorCallNode(name, code, argType, line(expr)) + .methodFullName(PhpOperators.cloneFunc) + + callAst(callNode, argAst :: Nil) + + private def astForEmpty(expr: PhpEmptyExpr): Ast = + val name = PhpOperators.emptyFunc + val argAst = astForExpr(expr.expr) + val code = s"$name(${argAst.rootCodeOrEmpty})" + + val callNode = + newOperatorCallNode( + name, + code, + typeFullName = Some(TypeConstants.Bool), + line = line(expr) + ) + .methodFullName(PhpOperators.emptyFunc) + + callAst(callNode, argAst :: Nil) + + private def astForEval(expr: PhpEvalExpr): Ast = + val name = PhpOperators.evalFunc + val argAst = astForExpr(expr.expr) + val code = s"$name(${argAst.rootCodeOrEmpty})" + + val callNode = + newOperatorCallNode( + name, + code, + typeFullName = Some(TypeConstants.Bool), + line = line(expr) + ) + .methodFullName(PhpOperators.evalFunc) + + callAst(callNode, argAst :: Nil) + + private def astForExit(expr: PhpExitExpr): Ast = + val name = PhpOperators.exitFunc + val args = expr.expr.map(astForExpr) + val code = s"$name(${args.map(_.rootCodeOrEmpty).getOrElse("")})" + + val callNode = newOperatorCallNode(name, code, Some(TypeConstants.Void), line(expr)) + .methodFullName(PhpOperators.exitFunc) + + callAst(callNode, args.toList) + + private def getTmpIdentifier( + originNode: PhpNode, + maybeTypeFullName: Option[String], + prefix: String = "" + ): NewIdentifier = + val name = s"$prefix${getNewTmpName()}" + val typeFullName = maybeTypeFullName.getOrElse(TypeConstants.Any) + identifierNode(originNode, name, s"$$$name", typeFullName) + + private def astForArrayExpr(expr: PhpArrayExpr): Ast = + val idxTracker = new ArrayIndexTracker + + val tmpIdentifier = getTmpIdentifier(expr, Some(TypeConstants.Array)) + + val itemAssignments = expr.items.flatMap { + case Some(item) => Option(assignForArrayItem(item, tmpIdentifier.name, idxTracker)) + case None => + idxTracker.next // Skip an index + None + } + val arrayBlock = blockNode(expr) + + Ast(arrayBlock) + .withChildren(itemAssignments) + .withChild(Ast(tmpIdentifier)) + + private def astForListExpr(expr: PhpListExpr): Ast = + /* TODO: Handling list in a way that will actually work with dataflow tracking is somewhat more complicated than + * this and will likely need a fairly ugly lowering. + * + * In short, the case: + * list($a, $b) = $arr; + * can be lowered to: + * $a = $arr[0]; + * $b = $arr[1]; + * + * the case: + * list("id" => $a, "name" => $b) = $arr; + * can be lowered to: + * $a = $arr["id"]; + * $b = $arr["name"]; + * + * and the case: + * foreach ($arr as list($a, $b)) { ... } + * can be lowered as above for each $arr[i]; + * + * The below is just a placeholder to prevent crashes while figuring out the cleanest way to + * implement the above lowering or to think of a better way to do it. + */ + + val name = PhpOperators.listFunc + val args = expr.items.flatten.map { item => astForExpr(item.value) } + val listCode = s"$name(${args.map(_.rootCodeOrEmpty).mkString(",")})" + val listNode = newOperatorCallNode(name, listCode, line = line(expr)) + .methodFullName(PhpOperators.listFunc) + + callAst(listNode, args) + end astForListExpr + + private def astForNewExpr(expr: PhpNewExpr): Ast = + expr.className match + case classLikeStmt: PhpClassLikeStmt => + astForAnonymousClassInstantiation(expr, classLikeStmt) + + case classNameExpr: PhpExpr => + astForSimpleNewExpr(expr, classNameExpr) + + case other => + throw new NotImplementedError( + s"unexpected expression '$other' of type ${other.getClass}" + ) + + private def astForMatchExpr(expr: PhpMatchExpr): Ast = + val conditionAst = astForExpr(expr.condition) + + val matchNode = controlStructureNode( + expr, + ControlStructureTypes.MATCH, + s"match (${conditionAst.rootCodeOrEmpty})" + ) + + val matchBodyBlock = blockNode(expr) + val armsAsts = expr.matchArms.flatMap(astsForMatchArm) + val matchBody = Ast(matchBodyBlock).withChildren(armsAsts) + + controlStructureAst(matchNode, Option(conditionAst), matchBody :: Nil) + + private def astsForMatchArm(matchArm: PhpMatchArm): List[Ast] = + // TODO Don't just throw away the condition asts here (also for switch cases) + val targets = matchArm.conditions.map { condition => + val conditionAst = astForExpr(condition) + // In PHP cases aren't labeled with `case`, but this is used by the CFG creator to differentiate between + // case/default labels and other labels. + val code = s"case ${conditionAst.rootCode.getOrElse(NameConstants.Unknown)}" + NewJumpTarget().name(code).code(code).lineNumber(line(condition)) + } + val defaultLabel = Option.when(matchArm.isDefault)( + NewJumpTarget().name(NameConstants.Default).code(NameConstants.Default).lineNumber(line( + matchArm + )) + ) + val targetAsts = (targets ++ defaultLabel.toList).map(Ast(_)) + + val bodyAst = astForExpr(matchArm.body) + + targetAsts :+ bodyAst + end astsForMatchArm + + private def astForYieldExpr(expr: PhpYieldExpr): Ast = + val maybeKey = expr.key.map(astForExpr) + val maybeVal = expr.value.map(astForExpr) + + val code = (maybeKey, maybeVal) match + case (Some(key), Some(value)) => + s"yield ${key.rootCodeOrEmpty} => ${value.rootCodeOrEmpty}" + + case _ => + s"yield ${maybeKey.map(_.rootCodeOrEmpty).getOrElse("")}${maybeVal.map(_.rootCodeOrEmpty).getOrElse("")}".trim + + val yieldNode = controlStructureNode(expr, ControlStructureTypes.YIELD, code) + + Ast(yieldNode) + .withChildren(maybeKey.toList) + .withChildren(maybeVal.toList) + + private def astForClosureExpr(closureExpr: PhpClosureExpr): Ast = + val methodName = scope.getScopedClosureName + val methodRef = methodRefNode(closureExpr, methodName, methodName, TypeConstants.Any) + + val localsForUses = closureExpr.uses.flatMap { closureUse => + val variableAst = astForExpr(closureUse.variable) + val codePref = if closureUse.byRef then "&" else "" + + variableAst.root match + case Some(identifier: NewIdentifier) => + // This is the expected case and is handled well + Some(localNode( + closureExpr, + identifier.name, + codePref ++ identifier.code, + TypeConstants.Any + )) + case Some(expr: ExpressionNew) => + // Results here may be bad, but its' the best we're likely to do + Some(localNode( + closureExpr, + expr.code, + codePref ++ expr.code, + TypeConstants.Any + )) + case Some(other) => + // This should never happen + logger.debug(s"Found ast '$other' for closure use in $filename") + None + case None => + // This should never happen + logger.debug(s"Found empty ast for closure use in $filename") + None + end match + } + + // Add closure bindings to diffgraph + localsForUses.foreach { local => + val closureBindingId = s"$filename:$methodName:${local.name}" + local.closureBindingId(closureBindingId) + scope.addToScope(local.name, local) + + val closureBindingNode = NewClosureBinding() + .closureBindingId(closureBindingId) + .closureOriginalName(local.name) + .evaluationStrategy(EvaluationStrategies.BY_SHARING) + + // The ref edge to the captured local is added in the ClosureRefPass + diffGraph.addNode(closureBindingNode) + diffGraph.addEdge(methodRef, closureBindingNode, EdgeTypes.CAPTURE) + } + + // Create method for closure + val name = PhpNameExpr(methodName, closureExpr.attributes) + // TODO Check for static modifier + val modifiers = + "LAMBDA" :: (if closureExpr.isStatic then ModifierTypes.STATIC :: Nil else Nil) + val methodDecl = PhpMethodDecl( + name, + closureExpr.params, + modifiers, + closureExpr.returnType, + closureExpr.stmts, + closureExpr.returnByRef, + namespacedName = None, + isClassMethod = closureExpr.isStatic, + closureExpr.attributes + ) + val methodAst = astForMethodDecl(methodDecl, localsForUses.map(Ast(_)), Option(methodName)) + + val usesCode = localsForUses match + case Nil => "" + case locals => s" use(${locals.map(_.code).mkString(", ")})" + methodAst.root.collect { case method: NewMethod => method }.foreach { methodNode => + methodNode.code(methodNode.code ++ usesCode) + } + + // Add method to scope to be attached to typeDecl later + scope.addAnonymousMethod(methodAst) + + Ast(methodRef) + end astForClosureExpr + + private def astForYieldFromExpr(expr: PhpYieldFromExpr): Ast = + // TODO This is currently only distinguishable from yield by the code field. Decide whether to treat YIELD_FROM + // separately or whether to lower this to a foreach with regular yields. + val exprAst = astForExpr(expr.expr) + + val code = s"yield from ${exprAst.rootCodeOrEmpty}" + + val yieldNode = controlStructureNode(expr, ControlStructureTypes.YIELD, code) + + Ast(yieldNode) + .withChild(exprAst) + + private def astForAnonymousClassInstantiation( + expr: PhpNewExpr, + classLikeStmt: PhpClassLikeStmt + ): Ast = + // TODO Do this along with other anonymous class support + Ast() + + private def astForSimpleNewExpr(expr: PhpNewExpr, classNameExpr: PhpExpr): Ast = + val (maybeNameAst, className) = classNameExpr match + case nameExpr: PhpNameExpr => + (None, nameExpr.name) + + case expr: PhpExpr => + val ast = astForExpr(expr) + // The name doesn't make sense in this case, but the AST will be more useful + val name = ast.rootCode.getOrElse(NameConstants.Unknown) + (Option(ast), name) + + val tmpIdentifier = getTmpIdentifier(expr, Option(className)) + + // Alloc assign + val allocCode = s"$className.()" + val allocNode = + newOperatorCallNode(Operators.alloc, allocCode, Option(className), line(expr)) + val allocAst = callAst(allocNode, base = maybeNameAst) + val allocAssignCode = s"${tmpIdentifier.code} = ${allocAst.rootCodeOrEmpty}" + val allocAssignNode = newOperatorCallNode( + Operators.assignment, + allocAssignCode, + Option(className), + line(expr) + ) + val allocAssignAst = callAst(allocAssignNode, Ast(tmpIdentifier) :: allocAst :: Nil) + + // Init node + val initArgs = expr.args.map(astForCallArg) + val initSignature = s"$UnresolvedSignature(${initArgs.size})" + val initFullName = s"$className$InstanceMethodDelimiter${ConstructorMethodName}" + val initCode = s"$initFullName(${initArgs.map(_.rootCodeOrEmpty).mkString(",")})" + val initCallNode = callNode( + expr, + initCode, + ConstructorMethodName, + initFullName, + DispatchTypes.DYNAMIC_DISPATCH, + Some(initSignature), + Some(TypeConstants.Any) + ) + val initReceiver = Ast(tmpIdentifier.copy) + val initCallAst = callAst(initCallNode, initArgs, base = Option(initReceiver)) + + // Return identifier + val returnIdentifierAst = Ast(tmpIdentifier.copy) + + Ast(blockNode(expr, "", TypeConstants.Any)) + .withChild(allocAssignAst) + .withChild(initCallAst) + .withChild(returnIdentifierAst) + end astForSimpleNewExpr + + private def dimensionFromSimpleScalar( + scalar: PhpSimpleScalar, + idxTracker: ArrayIndexTracker + ): PhpExpr = + val maybeIntValue = scalar match + case string: PhpString => + string.value + .drop(1) + .dropRight(1) + .toIntOption + + case number => number.value.toIntOption + + maybeIntValue match + case Some(intValue) => + idxTracker.updateValue(intValue) + PhpInt(intValue.toString, scalar.attributes) + + case None => + scalar + end dimensionFromSimpleScalar + private def assignForArrayItem( + item: PhpArrayItem, + name: String, + idxTracker: ArrayIndexTracker + ): Ast = + // It's perhaps a bit clumsy to reconstruct PhpExpr nodes here, but reuse astForArrayDimExpr for consistency + val variable = PhpVariable(PhpNameExpr(name, item.attributes), item.attributes) + + val dimension = item.key match + case Some(key: PhpSimpleScalar) => dimensionFromSimpleScalar(key, idxTracker) + case Some(key) => key + case None => PhpInt(idxTracker.next, item.attributes) + + val dimFetchNode = PhpArrayDimFetchExpr(variable, Option(dimension), item.attributes) + val dimFetchAst = astForArrayDimFetchExpr(dimFetchNode) + + val valueAst = astForArrayItemValue(item) + + val assignCode = s"${dimFetchAst.rootCodeOrEmpty} = ${valueAst.rootCodeOrEmpty}" + + val assignNode = newOperatorCallNode(Operators.assignment, assignCode, line = line(item)) + + callAst(assignNode, dimFetchAst :: valueAst :: Nil) + end assignForArrayItem + + private def astForArrayItemValue(item: PhpArrayItem): Ast = + val exprAst = astForExpr(item.value) + val valueCode = exprAst.rootCodeOrEmpty + + if item.byRef then + val parentCall = + newOperatorCallNode(Operators.addressOf, s"&$valueCode", line = line(item)) + callAst(parentCall, exprAst :: Nil) + else if item.unpack then + val parentCall = + newOperatorCallNode(PhpOperators.unpack, s"...$valueCode", line = line(item)) + callAst(parentCall, exprAst :: Nil) + else + exprAst + + private def astForArrayDimFetchExpr(expr: PhpArrayDimFetchExpr): Ast = + val variableAst = astForExpr(expr.variable) + val variableCode = variableAst.rootCodeOrEmpty + + expr.dimension match + case Some(dimension) => + val dimensionAst = astForExpr(dimension) + val code = s"$variableCode[${dimensionAst.rootCodeOrEmpty}]" + val accessNode = newOperatorCallNode(Operators.indexAccess, code, line = line(expr)) + callAst(accessNode, variableAst :: dimensionAst :: Nil) + + case None => + val errorPosition = s"${variableCode}:${line(expr).getOrElse("")}:${filename}" + logger.debug( + s"ArrayDimFetchExpr without dimensions should be handled in assignment: ${errorPosition}" + ) + Ast() + + private def astForErrorSuppressExpr(expr: PhpErrorSuppressExpr): Ast = + val childAst = astForExpr(expr.expr) + + val code = s"@${childAst.rootCodeOrEmpty}" + val suppressNode = newOperatorCallNode(PhpOperators.errorSuppress, code, line = line(expr)) + childAst.rootType.foreach(typ => suppressNode.typeFullName(typ)) + + callAst(suppressNode, childAst :: Nil) + + private def astForInstanceOfExpr(expr: PhpInstanceOfExpr): Ast = + val exprAst = astForExpr(expr.expr) + val classAst = astForExpr(expr.className) + + val code = s"${exprAst.rootCodeOrEmpty} instanceof ${classAst.rootCodeOrEmpty}" + val instanceOfNode = + newOperatorCallNode(Operators.instanceOf, code, Some(TypeConstants.Bool), line(expr)) + + callAst(instanceOfNode, exprAst :: classAst :: Nil) + + private def astForPropertyFetchExpr(expr: PhpPropertyFetchExpr): Ast = + val objExprAst = astForExpr(expr.expr) + + val fieldAst = expr.name match + case name: PhpNameExpr => Ast(newFieldIdentifierNode(name.name, line(expr))) + case other => astForExpr(other) + + val accessSymbol = + if expr.isStatic then + "::" + else if expr.isNullsafe then + "?->" + else + "->" + + val code = s"${objExprAst.rootCodeOrEmpty}$accessSymbol${fieldAst.rootCodeOrEmpty}" + val fieldAccessNode = newOperatorCallNode(Operators.fieldAccess, code, line = line(expr)) + + callAst(fieldAccessNode, objExprAst :: fieldAst :: Nil) + end astForPropertyFetchExpr + + private def astForIncludeExpr(expr: PhpIncludeExpr): Ast = + val exprAst = astForExpr(expr.expr) + val code = s"${expr.includeType} ${exprAst.rootCodeOrEmpty}" + val callNode = newOperatorCallNode(expr.includeType, code, line = line(expr)) + + callAst(callNode, exprAst :: Nil) + + private def astForShellExecExpr(expr: PhpShellExecExpr): Ast = + val args = astForEncapsed(expr.parts) + val code = "`" + args.rootCodeOrEmpty + "`" + + val callNode = newOperatorCallNode(PhpOperators.shellExec, code, line = line(expr)) + + callAst(callNode, args :: Nil) + + private def astForMagicClassConstant(expr: PhpClassConstFetchExpr): Ast = + val classAst = astForExpr(expr.className) + val typeFullName = expr.className match + case nameExpr: PhpNameExpr => + scope + .lookupVariable(nameExpr.name) + .flatMap(_.properties.get(PropertyNames.TYPE_FULL_NAME).map(_.toString)) + .getOrElse(nameExpr.name) + + case expr => + classAst.rootType.orElse(classAst.rootName).getOrElse(UnresolvedNamespace) + + Ast(typeRefNode(expr, classAst.rootCodeOrEmpty, typeFullName)) + + private def astForClassConstFetchExpr(expr: PhpClassConstFetchExpr): Ast = + expr.constantName match + // Foo::class should be a TypeRef and not a field access + case Some(constNameExpr) if constNameExpr.name == NameConstants.Class => + astForMagicClassConstant(expr) + + case _ => + val targetAst = astForExpr(expr.className) + val fieldIdentifierName = + expr.constantName.map(_.name).getOrElse(NameConstants.Unknown) + val fieldIdentifier = newFieldIdentifierNode(fieldIdentifierName, line(expr)) + val fieldAccessCode = s"${targetAst.rootCodeOrEmpty}::${fieldIdentifier.code}" + val fieldAccessCall = + newOperatorCallNode(Operators.fieldAccess, fieldAccessCode, line = line(expr)) + callAst(fieldAccessCall, List(targetAst, Ast(fieldIdentifier))) + + private def astForConstFetchExpr(expr: PhpConstFetchExpr): Ast = + val constName = expr.name.name + + if NameConstants.isBoolean(constName) then + Ast(literalNode(expr, constName, TypeConstants.Bool)) + else if NameConstants.isNull(constName) then + Ast(literalNode(expr, constName, TypeConstants.NullType)) + else + val namespaceName = NamespaceTraversal.globalNamespaceName + val identifier = identifierNode(expr, namespaceName, namespaceName, "ANY") + val fieldIdentifier = newFieldIdentifierNode(constName, line = line(expr)) + + val fieldAccessNode = + newOperatorCallNode(Operators.fieldAccess, code = constName, line = line(expr)) + val args = List(identifier, fieldIdentifier).map(Ast(_)) + + callAst(fieldAccessNode, args) + + protected def line(phpNode: PhpNode): Option[Integer] = phpNode.attributes.lineNumber + protected def column(phpNode: PhpNode): Option[Integer] = None + protected def lineEnd(phpNode: PhpNode): Option[Integer] = None + protected def columnEnd(phpNode: PhpNode): Option[Integer] = None + protected def code(phpNode: PhpNode): String = + "" // Sadly, the Php AST does not carry any code fields +end AstCreator + +object AstCreator: + object TypeConstants: + val String: String = "string" + val Int: String = "int" + val Float: String = "float" + val Bool: String = "bool" + val Void: String = "void" + val Any: String = "ANY" + val Array: String = "array" + val NullType: String = "null" + val VariadicPlaceholder: String = "PhpVariadicPlaceholder" + + object NameConstants: + val Default: String = "default" + val HaltCompiler: String = "__halt_compiler" + val This: String = "this" + val Unknown: String = "UNKNOWN" + val Closure: String = "__closure" + val Class: String = "class" + val True: String = "true"; + val False: String = "false"; + val NullName: String = "null"; + + def isBoolean(name: String): Boolean = + List(True, False).contains(name) + + def isNull(name: String): Boolean = + name.toLowerCase == NullName + + val operatorSymbols: Map[String, String] = Map( + Operators.and -> "&", + Operators.or -> "|", + Operators.xor -> "^", + Operators.logicalAnd -> "&&", + Operators.logicalOr -> "||", + PhpOperators.coalesceOp -> "??", + PhpOperators.concatOp -> ".", + Operators.division -> "/", + Operators.equals -> "==", + Operators.greaterEqualsThan -> ">=", + Operators.greaterThan -> ">", + PhpOperators.identicalOp -> "===", + PhpOperators.logicalXorOp -> "xor", + Operators.minus -> "-", + Operators.modulo -> "%", + Operators.multiplication -> "*", + Operators.notEquals -> "!=", + PhpOperators.notIdenticalOp -> "!==", + Operators.plus -> "+", + Operators.exponentiation -> "**", + Operators.shiftLeft -> "<<", + Operators.arithmeticShiftRight -> ">>", + Operators.lessEqualsThan -> "<=", + Operators.lessThan -> "<", + PhpOperators.spaceshipOp -> "<=>", + Operators.not -> "~", + Operators.logicalNot -> "!", + Operators.postDecrement -> "--", + Operators.postIncrement -> "++", + Operators.preDecrement -> "--", + Operators.preIncrement -> "++", + Operators.minus -> "-", + Operators.plus -> "+", + Operators.assignment -> "=", + Operators.assignmentAnd -> "&=", + Operators.assignmentOr -> "|=", + Operators.assignmentXor -> "^=", + PhpOperators.assignmentCoalesceOp -> "??=", + PhpOperators.assignmentConcatOp -> ".=", + Operators.assignmentDivision -> "/=", + Operators.assignmentMinus -> "-=", + Operators.assignmentModulo -> "%=", + Operators.assignmentMultiplication -> "*=", + Operators.assignmentPlus -> "+=", + Operators.assignmentExponentiation -> "**=", + Operators.assignmentShiftLeft -> "<<=", + Operators.assignmentArithmeticShiftRight -> ">>=" + ) +end AstCreator diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/astcreation/PhpBuiltins.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/astcreation/PhpBuiltins.scala new file mode 100644 index 00000000..52770dcc --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/astcreation/PhpBuiltins.scala @@ -0,0 +1,9 @@ +package io.appthreat.php2atom.astcreation + +import io.shiftleft.utils.IOUtils + +import scala.io.Source + +object PhpBuiltins: + lazy val FuncNames: Set[String] = + Source.fromResource("builtin_functions.txt").getLines().toSet diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/Domain.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/Domain.scala new file mode 100644 index 00000000..fd7f08b1 --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/Domain.scala @@ -0,0 +1,1442 @@ +package io.appthreat.php2atom.parser + +import io.appthreat.php2atom.astcreation.PhpBuiltins +import io.appthreat.php2atom.astcreation.AstCreator.TypeConstants +import io.appthreat.php2atom.parser.Domain.PhpAssignment.{AssignTypeMap, isAssignType} +import io.appthreat.php2atom.parser.Domain.PhpBinaryOp.{BinaryOpTypeMap, isBinaryOpType} +import io.appthreat.php2atom.parser.Domain.PhpCast.{CastTypeMap, isCastType} +import io.appthreat.php2atom.parser.Domain.PhpUnaryOp.{UnaryOpTypeMap, isUnaryOpType} +import io.appthreat.php2atom.parser.Domain.PhpUseType.{PhpUseType, getUseType} +import io.appthreat.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.{ModifierTypes, Operators} +import org.slf4j.LoggerFactory +import ujson.{Arr, Obj, Str, Value} + +import scala.util.{Success, Try} +import io.appthreat.php2atom.astcreation.AstCreator + +object Domain: + + object PhpOperators: + // TODO Decide which of these should be moved to codepropertygraph + val coalesceOp = ".coalesce" + val concatOp = ".concat" + val identicalOp = ".identical" + val logicalXorOp = ".logicalXor" + val notIdenticalOp = ".notIdentical" + val spaceshipOp = ".spaceship" + val elvisOp = ".elvis" + val unpack = ".unpack" + // Used for $array[] = $var type assignments + val emptyArrayIdx = ".emptyArrayIdx" + val errorSuppress = ".errorSuppress" + // Double arrow operator used to represent key/value pairs: key => value + val doubleArrow = ".doubleArrow" + + val assignmentCoalesceOp = ".assignmentCoalesce" + val assignmentConcatOp = ".assignmentConcat" + + val encaps = "encaps" + val declareFunc = "declare" + val global = "global" + + // These are handled as special cases for builtins since they have separate AST nodes in the PHP-parser output. + val issetFunc = s"isset" + val printFunc = s"print" + val cloneFunc = s"clone" + val emptyFunc = s"empty" + val evalFunc = s"eval" + val exitFunc = s"exit" + // Used for multiple assignments for example `list($a, $b) = $someArray` + val listFunc = s"list" + val isNull = s"is_null" + val unset = s"unset" + val shellExec = s"shell_exec" + end PhpOperators + + object PhpDomainTypeConstants: + val array = "array" + val bool = "bool" + val double = "double" + val int = "int" + val obj = "object" + val string = "string" + val unset = "unset" + + private val logger = LoggerFactory.getLogger(Domain.getClass) + val NamespaceDelimiter = "\\" + val StaticMethodDelimiter = "::" + val InstanceMethodDelimiter = "->" + // Used for creating the default constructor. + val ConstructorMethodName = "__construct" + + final case class PhpAttributes(lineNumber: Option[Integer], kind: Option[Int]) + object PhpAttributes: + val Empty: PhpAttributes = PhpAttributes(None, None) + + def apply(json: Value): PhpAttributes = + Try(json("attributes")) match + case Success(Obj(attributes)) => + val startLine = + attributes.get("startLine").map(num => Integer.valueOf(num.num.toInt)) + val kind = attributes.get("kind").map(_.num.toInt) + PhpAttributes(startLine, kind) + + case Success(Arr(_)) => + logger.debug(s"Found array attributes in $json") + PhpAttributes.Empty + + case unhandled => + logger.warn(s"Could not find attributes object in type $unhandled") + PhpAttributes.Empty + + object PhpModifiers: + private val ModifierMasks = List( + (1, ModifierTypes.PUBLIC), + (2, ModifierTypes.PROTECTED), + (4, ModifierTypes.PRIVATE), + (8, ModifierTypes.STATIC), + (16, ModifierTypes.ABSTRACT), + (32, ModifierTypes.FINAL), + (64, ModifierTypes.READONLY) + ) + + private val AccessModifiers: Set[String] = + Set(ModifierTypes.PUBLIC, ModifierTypes.PROTECTED, ModifierTypes.PRIVATE) + + def containsAccessModifier(modifiers: List[String]): Boolean = + modifiers.toSet.intersect(AccessModifiers).nonEmpty + + def getModifierSet(json: Value, modifierString: String = "flags"): List[String] = + val flags = json.objOpt.flatMap(_.get(modifierString)).map(_.num.toInt).getOrElse(0) + ModifierMasks.collect { + case (mask, typ) if (flags & mask) != 0 => typ + } + end PhpModifiers + + sealed trait PhpNode: + def attributes: PhpAttributes + + final case class PhpFile(children: List[PhpStmt]) extends PhpNode: + override val attributes: PhpAttributes = PhpAttributes.Empty + + final case class PhpParam( + name: String, + paramType: Option[PhpNameExpr], + byRef: Boolean, + isVariadic: Boolean, + default: Option[PhpExpr], + // TODO type + flags: Int, + // TODO attributeGroups: Seq[PhpAttributeGroup], + attributes: PhpAttributes + ) extends PhpNode + + sealed trait PhpArgument extends PhpNode + final case class PhpArg( + expr: PhpExpr, + parameterName: Option[String], + byRef: Boolean, + unpack: Boolean, + attributes: PhpAttributes + ) extends PhpArgument + object PhpArg: + def apply(expr: PhpExpr): PhpArg = + PhpArg( + expr, + parameterName = None, + byRef = false, + unpack = false, + attributes = expr.attributes + ) + final case class PhpVariadicPlaceholder(attributes: Domain.PhpAttributes) extends PhpArgument + + sealed trait PhpStmt extends PhpNode + sealed trait PhpStmtWithBody extends PhpStmt: + def stmts: List[PhpStmt] + + // In the PhpParser output, comments are included as an attribute to the first statement following the comment. If + // no such statement exists, a Nop statement (which does not exist in PHP) is added as a sort of comment container. + final case class NopStmt(attributes: PhpAttributes) extends PhpStmt + final case class PhpEchoStmt(exprs: Seq[PhpExpr], attributes: PhpAttributes) extends PhpStmt + final case class PhpBreakStmt(num: Option[Int], attributes: PhpAttributes) extends PhpStmt + final case class PhpContinueStmt(num: Option[Int], attributes: PhpAttributes) extends PhpStmt + final case class PhpWhileStmt(cond: PhpExpr, stmts: List[PhpStmt], attributes: PhpAttributes) + extends PhpStmtWithBody + final case class PhpDoStmt(cond: PhpExpr, stmts: List[PhpStmt], attributes: PhpAttributes) + extends PhpStmtWithBody + final case class PhpForStmt( + inits: List[PhpExpr], + conditions: List[PhpExpr], + loopExprs: List[PhpExpr], + stmts: List[PhpStmt], + attributes: PhpAttributes + ) extends PhpStmtWithBody + final case class PhpIfStmt( + cond: PhpExpr, + stmts: List[PhpStmt], + elseIfs: List[PhpElseIfStmt], + elseStmt: Option[PhpElseStmt], + attributes: PhpAttributes + ) extends PhpStmtWithBody + final case class PhpElseIfStmt(cond: PhpExpr, stmts: List[PhpStmt], attributes: PhpAttributes) + extends PhpStmtWithBody + final case class PhpElseStmt(stmts: List[PhpStmt], attributes: PhpAttributes) + extends PhpStmtWithBody + final case class PhpSwitchStmt( + condition: PhpExpr, + cases: List[PhpCaseStmt], + attributes: PhpAttributes + ) extends PhpStmt + final case class PhpCaseStmt( + condition: Option[PhpExpr], + stmts: List[PhpStmt], + attributes: PhpAttributes + ) extends PhpStmtWithBody + final case class PhpTryStmt( + stmts: List[PhpStmt], + catches: List[PhpCatchStmt], + finallyStmt: Option[PhpFinallyStmt], + attributes: PhpAttributes + ) extends PhpStmtWithBody + final case class PhpCatchStmt( + types: List[PhpNameExpr], + variable: Option[PhpExpr], + stmts: List[PhpStmt], + attributes: PhpAttributes + ) extends PhpStmtWithBody + final case class PhpFinallyStmt(stmts: List[PhpStmt], attributes: PhpAttributes) + extends PhpStmtWithBody + final case class PhpReturnStmt(expr: Option[PhpExpr], attributes: PhpAttributes) extends PhpStmt + + final case class PhpMethodDecl( + name: PhpNameExpr, + params: Seq[PhpParam], + modifiers: List[String], + returnType: Option[PhpNameExpr], + stmts: List[PhpStmt], + returnByRef: Boolean, + // TODO attributeGroups: Seq[PhpAttributeGroup], + namespacedName: Option[PhpNameExpr], + isClassMethod: Boolean, + attributes: PhpAttributes + ) extends PhpStmtWithBody + + final case class PhpClassLikeStmt( + name: Option[PhpNameExpr], + modifiers: List[String], + extendsNames: List[PhpNameExpr], + implementedInterfaces: List[PhpNameExpr], + stmts: List[PhpStmt], + classLikeType: String, + // Optionally used for enums with values + scalarType: Option[PhpNameExpr], + hasConstructor: Boolean, + attributes: PhpAttributes + ) extends PhpStmtWithBody + object ClassLikeTypes: + val Class: String = "class" + val Trait: String = "trait" + val Interface: String = "interface" + val Enum: String = "enum" + + final case class PhpEnumCaseStmt( + name: PhpNameExpr, + expr: Option[PhpExpr], + attributes: PhpAttributes + ) extends PhpStmt + + final case class PhpPropertyStmt( + modifiers: List[String], + variables: List[PhpPropertyValue], + typeName: Option[PhpNameExpr], + attributes: PhpAttributes + ) extends PhpStmt + + final case class PhpPropertyValue( + name: PhpNameExpr, + defaultValue: Option[PhpExpr], + attributes: PhpAttributes + ) extends PhpStmt + + final case class PhpConstStmt( + modifiers: List[String], + consts: List[PhpConstDeclaration], + attributes: PhpAttributes + ) extends PhpStmt + + final case class PhpGotoStmt(label: PhpNameExpr, attributes: PhpAttributes) extends PhpStmt + final case class PhpLabelStmt(label: PhpNameExpr, attributes: PhpAttributes) extends PhpStmt + final case class PhpHaltCompilerStmt(attributes: PhpAttributes) extends PhpStmt + + final case class PhpConstDeclaration( + name: PhpNameExpr, + value: PhpExpr, + namespacedName: Option[PhpNameExpr], + attributes: PhpAttributes + ) extends PhpStmt + + final case class PhpNamespaceStmt( + name: Option[PhpNameExpr], + stmts: List[PhpStmt], + attributes: PhpAttributes + ) extends PhpStmtWithBody + + final case class PhpDeclareStmt( + declares: Seq[PhpDeclareItem], + stmts: Option[List[PhpStmt]], + attributes: PhpAttributes + ) extends PhpStmt + final case class PhpDeclareItem(key: PhpNameExpr, value: PhpExpr, attributes: PhpAttributes) + extends PhpStmt + + final case class PhpUnsetStmt(vars: List[PhpExpr], attributes: PhpAttributes) extends PhpStmt + + final case class PhpStaticStmt(vars: List[PhpStaticVar], attributes: PhpAttributes) + extends PhpStmt + + final case class PhpStaticVar( + variable: PhpVariable, + defaultValue: Option[PhpExpr], + attributes: PhpAttributes + ) extends PhpStmt + + final case class PhpGlobalStmt(vars: List[PhpExpr], attributes: PhpAttributes) extends PhpStmt + + final case class PhpUseStmt( + uses: List[PhpUseUse], + useType: PhpUseType, + attributes: PhpAttributes + ) extends PhpStmt + final case class PhpGroupUseStmt( + prefix: PhpNameExpr, + uses: List[PhpUseUse], + useType: PhpUseType, + attributes: PhpAttributes + ) extends PhpStmt + final case class PhpUseUse( + originalName: PhpNameExpr, + alias: Option[PhpNameExpr], + useType: PhpUseType, + attributes: PhpAttributes + ) extends PhpStmt + + case object PhpUseType: + sealed trait PhpUseType + case object Unknown extends PhpUseType + case object Normal extends PhpUseType + case object Function extends PhpUseType + case object Constant extends PhpUseType + + def getUseType(typeNum: Int): PhpUseType = + typeNum match + case 1 => Normal + case 2 => Function + case 3 => Constant + case _ => Unknown + + final case class PhpForeachStmt( + iterExpr: PhpExpr, + keyVar: Option[PhpExpr], + valueVar: PhpExpr, + assignByRef: Boolean, + stmts: List[PhpStmt], + attributes: PhpAttributes + ) extends PhpStmtWithBody + final case class PhpTraitUseStmt( + traits: List[PhpNameExpr], + adaptations: List[PhpTraitUseAdaptation], + attributes: PhpAttributes + ) extends PhpStmt + sealed trait PhpTraitUseAdaptation extends PhpStmt + final case class PhpPrecedenceAdaptation( + traitName: PhpNameExpr, + methodName: PhpNameExpr, + insteadOf: List[PhpNameExpr], + attributes: PhpAttributes + ) extends PhpTraitUseAdaptation + final case class PhpAliasAdaptation( + traitName: Option[PhpNameExpr], + methodName: PhpNameExpr, + newModifier: Option[String], + newName: Option[PhpNameExpr], + attributes: PhpAttributes + ) extends PhpTraitUseAdaptation + + sealed trait PhpExpr extends PhpStmt + + final case class PhpNewExpr( + className: PhpNode, + args: List[PhpArgument], + attributes: PhpAttributes + ) extends PhpExpr + + final case class PhpIncludeExpr(expr: PhpExpr, includeType: String, attributes: PhpAttributes) + extends PhpExpr + case object PhpIncludeType: + val Include: String = "include" + val IncludeOnce: String = "include_once" + val Require: String = "require" + val RequireOnce: String = "require_once" + + final case class PhpCallExpr( + target: Option[PhpExpr], + methodName: PhpExpr, + args: Seq[PhpArgument], + isNullSafe: Boolean, + isStatic: Boolean, + attributes: PhpAttributes + ) extends PhpExpr + final case class PhpVariable(value: PhpExpr, attributes: PhpAttributes) extends PhpExpr + final case class PhpNameExpr(name: String, attributes: PhpAttributes) extends PhpExpr + final case class PhpCloneExpr(expr: PhpExpr, attributes: PhpAttributes) extends PhpExpr + final case class PhpEmptyExpr(expr: PhpExpr, attributes: PhpAttributes) extends PhpExpr + final case class PhpEvalExpr(expr: PhpExpr, attributes: PhpAttributes) extends PhpExpr + final case class PhpExitExpr(expr: Option[PhpExpr], attributes: PhpAttributes) extends PhpExpr + final case class PhpBinaryOp( + operator: String, + left: PhpExpr, + right: PhpExpr, + attributes: PhpAttributes + ) extends PhpExpr + object PhpBinaryOp: + val BinaryOpTypeMap: Map[String, String] = Map( + "Expr_BinaryOp_BitwiseAnd" -> Operators.and, + "Expr_BinaryOp_BitwiseOr" -> Operators.or, + "Expr_BinaryOp_BitwiseXor" -> Operators.xor, + "Expr_BinaryOp_BooleanAnd" -> Operators.logicalAnd, + "Expr_BinaryOp_BooleanOr" -> Operators.logicalOr, + "Expr_BinaryOp_Coalesce" -> PhpOperators.coalesceOp, + "Expr_BinaryOp_Concat" -> PhpOperators.concatOp, + "Expr_BinaryOp_Div" -> Operators.division, + "Expr_BinaryOp_Equal" -> Operators.equals, + "Expr_BinaryOp_GreaterOrEqual" -> Operators.greaterEqualsThan, + "Expr_BinaryOp_Greater" -> Operators.greaterThan, + "Expr_BinaryOp_Identical" -> PhpOperators.identicalOp, + "Expr_BinaryOp_LogicalAnd" -> Operators.logicalAnd, + "Expr_BinaryOp_LogicalOr" -> Operators.logicalOr, + "Expr_BinaryOp_LogicalXor" -> PhpOperators.logicalXorOp, + "Expr_BinaryOp_Minus" -> Operators.minus, + "Expr_BinaryOp_Mod" -> Operators.modulo, + "Expr_BinaryOp_Mul" -> Operators.multiplication, + "Expr_BinaryOp_NotEqual" -> Operators.notEquals, + "Expr_BinaryOp_NotIdentical" -> PhpOperators.notIdenticalOp, + "Expr_BinaryOp_Plus" -> Operators.plus, + "Expr_BinaryOp_Pow" -> Operators.exponentiation, + "Expr_BinaryOp_ShiftLeft" -> Operators.shiftLeft, + "Expr_BinaryOp_ShiftRight" -> Operators.arithmeticShiftRight, + "Expr_BinaryOp_SmallerOrEqual" -> Operators.lessEqualsThan, + "Expr_BinaryOp_Smaller" -> Operators.lessThan, + "Expr_BinaryOp_Spaceship" -> PhpOperators.spaceshipOp + ) + + def isBinaryOpType(typeName: String): Boolean = + BinaryOpTypeMap.contains(typeName) + end PhpBinaryOp + final case class PhpUnaryOp(operator: String, expr: PhpExpr, attributes: PhpAttributes) + extends PhpExpr + object PhpUnaryOp: + val UnaryOpTypeMap: Map[String, String] = Map( + "Expr_BitwiseNot" -> Operators.not, + "Expr_BooleanNot" -> Operators.logicalNot, + "Expr_PostDec" -> Operators.postDecrement, + "Expr_PostInc" -> Operators.postIncrement, + "Expr_PreDec" -> Operators.preDecrement, + "Expr_PreInc" -> Operators.preIncrement, + "Expr_UnaryMinus" -> Operators.minus, + "Expr_UnaryPlus" -> Operators.plus + ) + + def isUnaryOpType(typeName: String): Boolean = + UnaryOpTypeMap.contains(typeName) + final case class PhpTernaryOp( + condition: PhpExpr, + thenExpr: Option[PhpExpr], + elseExpr: PhpExpr, + attributes: PhpAttributes + ) extends PhpExpr + + object PhpAssignment: + val AssignTypeMap: Map[String, String] = Map( + "Expr_Assign" -> Operators.assignment, + "Expr_AssignRef" -> Operators.assignment, + "Expr_AssignOp_BitwiseAnd" -> Operators.assignmentAnd, + "Expr_AssignOp_BitwiseOr" -> Operators.assignmentOr, + "Expr_AssignOp_BitwiseXor" -> Operators.assignmentXor, + "Expr_AssignOp_Coalesce" -> PhpOperators.assignmentCoalesceOp, + "Expr_AssignOp_Concat" -> PhpOperators.assignmentConcatOp, + "Expr_AssignOp_Div" -> Operators.assignmentDivision, + "Expr_AssignOp_Minus" -> Operators.assignmentMinus, + "Expr_AssignOp_Mod" -> Operators.assignmentModulo, + "Expr_AssignOp_Mul" -> Operators.assignmentMultiplication, + "Expr_AssignOp_Plus" -> Operators.assignmentPlus, + "Expr_AssignOp_Pow" -> Operators.assignmentExponentiation, + "Expr_AssignOp_ShiftLeft" -> Operators.assignmentShiftLeft, + "Expr_AssignOp_ShiftRight" -> Operators.assignmentArithmeticShiftRight + ) + + def isAssignType(typeName: String): Boolean = + AssignTypeMap.contains(typeName) + end PhpAssignment + final case class PhpAssignment( + assignOp: String, + target: PhpExpr, + source: PhpExpr, + isRefAssign: Boolean, + attributes: PhpAttributes + ) extends PhpExpr + + final case class PhpCast(typ: String, expr: PhpExpr, attributes: PhpAttributes) extends PhpExpr + object PhpCast: + val CastTypeMap: Map[String, String] = Map( + "Expr_Cast_Array" -> PhpDomainTypeConstants.array, + "Expr_Cast_Bool" -> PhpDomainTypeConstants.bool, + "Expr_Cast_Double" -> PhpDomainTypeConstants.double, + "Expr_Cast_Int" -> PhpDomainTypeConstants.int, + "Expr_Cast_Object" -> PhpDomainTypeConstants.obj, + "Expr_Cast_String" -> PhpDomainTypeConstants.string, + "Expr_Cast_Unset" -> PhpDomainTypeConstants.unset + ) + + def isCastType(typeName: String): Boolean = + CastTypeMap.contains(typeName) + + final case class PhpIsset(vars: Seq[PhpExpr], attributes: PhpAttributes) extends PhpExpr + final case class PhpPrint(expr: PhpExpr, attributes: PhpAttributes) extends PhpExpr + + sealed trait PhpScalar extends PhpExpr + sealed abstract class PhpSimpleScalar(val typeFullName: String) extends PhpScalar: + def value: String + def attributes: PhpAttributes + + final case class PhpString(val value: String, val attributes: PhpAttributes) + extends PhpSimpleScalar(TypeConstants.String) + object PhpString: + def withQuotes(value: String, attributes: PhpAttributes): PhpString = + PhpString(s"\"${escapeString(value)}\"", attributes) + + final case class PhpInt(val value: String, val attributes: PhpAttributes) + extends PhpSimpleScalar(TypeConstants.Int) + + final case class PhpFloat(val value: String, val attributes: PhpAttributes) + extends PhpSimpleScalar(TypeConstants.Float) + + final case class PhpEncapsed(parts: Seq[PhpExpr], attributes: PhpAttributes) extends PhpScalar + + final case class PhpThrowExpr(expr: PhpExpr, attributes: PhpAttributes) extends PhpExpr + final case class PhpListExpr(items: List[Option[PhpArrayItem]], attributes: PhpAttributes) + extends PhpExpr + + final case class PhpClassConstFetchExpr( + className: PhpExpr, + constantName: Option[PhpNameExpr], + attributes: PhpAttributes + ) extends PhpExpr + + final case class PhpConstFetchExpr(name: PhpNameExpr, attributes: PhpAttributes) extends PhpExpr + + final case class PhpArrayExpr(items: List[Option[PhpArrayItem]], attributes: PhpAttributes) + extends PhpExpr + final case class PhpArrayItem( + key: Option[PhpExpr], + value: PhpExpr, + byRef: Boolean, + unpack: Boolean, + attributes: PhpAttributes + ) extends PhpExpr + final case class PhpArrayDimFetchExpr( + variable: PhpExpr, + dimension: Option[PhpExpr], + attributes: PhpAttributes + ) extends PhpExpr + + final case class PhpErrorSuppressExpr(expr: PhpExpr, attributes: PhpAttributes) extends PhpExpr + + final case class PhpInstanceOfExpr(expr: PhpExpr, className: PhpExpr, attributes: PhpAttributes) + extends PhpExpr + + final case class PhpShellExecExpr(parts: PhpEncapsed, attributes: PhpAttributes) extends PhpExpr + + final case class PhpPropertyFetchExpr( + expr: PhpExpr, + name: PhpExpr, + isNullsafe: Boolean, + isStatic: Boolean, + attributes: PhpAttributes + ) extends PhpExpr + + final case class PhpMatchExpr( + condition: PhpExpr, + matchArms: List[PhpMatchArm], + attributes: PhpAttributes + ) extends PhpExpr + + final case class PhpMatchArm( + conditions: List[PhpExpr], + body: PhpExpr, + isDefault: Boolean, + attributes: PhpAttributes + ) extends PhpExpr + + final case class PhpYieldExpr( + key: Option[PhpExpr], + value: Option[PhpExpr], + attributes: PhpAttributes + ) extends PhpExpr + final case class PhpYieldFromExpr(expr: PhpExpr, attributes: PhpAttributes) extends PhpExpr + + final case class PhpClosureExpr( + params: List[PhpParam], + stmts: List[PhpStmt], + returnType: Option[PhpNameExpr], + uses: List[PhpClosureUse], + isStatic: Boolean, + returnByRef: Boolean, + isArrowFunc: Boolean, + attributes: PhpAttributes + ) extends PhpExpr + final case class PhpClosureUse(variable: PhpExpr, byRef: Boolean, attributes: PhpAttributes) + extends PhpExpr + + private def escapeString(value: String): String = + value + .replace("\\", "\\\\") + .replace("\n", "\\n") + .replace("\b", "\\b") + .replace("\r", "\\r") + .replace("\t", "\\t") + .replace("\'", "\\'") + .replace("\f", "\\f") + .replace("\"", "\\\"") + + private def readFile(json: Value): PhpFile = + json match + case arr: Arr => + val children = arr.value.map(readStmt).toList + PhpFile(children) + case unhandled => + logger.error( + s"Found unhandled type in readFile: ${unhandled.getClass} with value $unhandled" + ) + ??? + + private def readStmt(json: Value): PhpStmt = + json("nodeType").str match + case "Stmt_Echo" => + val values = json("exprs").arr.map(readExpr).toSeq + PhpEchoStmt(values, PhpAttributes(json)) + case "Stmt_Expression" => readExpr(json("expr")) + case "Stmt_Function" => readFunction(json) + case "Stmt_InlineHTML" => readInlineHtml(json) + case "Stmt_Break" => readBreak(json) + case "Stmt_Continue" => readContinue(json) + case "Stmt_While" => readWhile(json) + case "Stmt_Do" => readDo(json) + case "Stmt_For" => readFor(json) + case "Stmt_If" => readIf(json) + case "Stmt_Switch" => readSwitch(json) + case "Stmt_TryCatch" => readTry(json) + case "Stmt_Throw" => readThrow(json) + case "Stmt_Return" => readReturn(json) + case "Stmt_Class" => readClassLike(json, ClassLikeTypes.Class) + case "Stmt_Interface" => readClassLike(json, ClassLikeTypes.Interface) + case "Stmt_Trait" => readClassLike(json, ClassLikeTypes.Trait) + case "Stmt_Enum" => readClassLike(json, ClassLikeTypes.Enum) + case "Stmt_EnumCase" => readEnumCase(json) + case "Stmt_ClassMethod" => readClassMethod(json) + case "Stmt_Property" => readProperty(json) + case "Stmt_ClassConst" => readConst(json) + case "Stmt_Const" => readConst(json) + case "Stmt_Goto" => readGoto(json) + case "Stmt_Label" => readLabel(json) + case "Stmt_HaltCompiler" => readHaltCompiler(json) + case "Stmt_Namespace" => readNamespace(json) + case "Stmt_Nop" => NopStmt(PhpAttributes(json)) + case "Stmt_Declare" => readDeclare(json) + case "Stmt_Unset" => readUnset(json) + case "Stmt_Static" => readStatic(json) + case "Stmt_Global" => readGlobal(json) + case "Stmt_Use" => readUse(json) + case "Stmt_GroupUse" => readGroupUse(json) + case "Stmt_Foreach" => readForeach(json) + case "Stmt_TraitUse" => readTraitUse(json) + case unhandled => + logger.error(s"Found unhandled stmt type: $unhandled") + ??? + + private def readString(json: Value): PhpString = + PhpString.withQuotes(json("value").str, PhpAttributes(json)) + + private def readInlineHtml(json: Value): PhpStmt = + val value = readString(json) + PhpEchoStmt(List(value), value.attributes) + + private def readBreakContinueNum(json: Value): Option[Int] = + Option.unless(json("num").isNull)(json("num")("value").toString).flatMap(_.toIntOption) + private def readBreak(json: Value): PhpBreakStmt = + val num = readBreakContinueNum(json) + PhpBreakStmt(num, PhpAttributes(json)) + + private def readContinue(json: Value): PhpContinueStmt = + val num = readBreakContinueNum(json) + PhpContinueStmt(num, PhpAttributes(json)) + + private def readWhile(json: Value): PhpWhileStmt = + val cond = readExpr(json("cond")) + val stmts = json("stmts").arr.toList.map(readStmt) + PhpWhileStmt(cond, stmts, PhpAttributes(json)) + + private def readDo(json: Value): PhpDoStmt = + val cond = readExpr(json("cond")) + val stmts = json("stmts").arr.toList.map(readStmt) + PhpDoStmt(cond, stmts, PhpAttributes(json)) + + private def readFor(json: Value): PhpForStmt = + val inits = json("init").arr.map(readExpr).toList + val conditions = json("cond").arr.map(readExpr).toList + val loopExprs = json("loop").arr.map(readExpr).toList + val bodyStmts = json("stmts").arr.map(readStmt).toList + + PhpForStmt(inits, conditions, loopExprs, bodyStmts, PhpAttributes(json)) + + private def readIf(json: Value): PhpIfStmt = + val condition = readExpr(json("cond")) + val stmts = json("stmts").arr.map(readStmt).toList + val elseIfs = json("elseifs").arr.map(readElseIf).toList + val elseStmt = Option.when(!json("else").isNull)(readElse(json("else"))) + + PhpIfStmt(condition, stmts, elseIfs, elseStmt, PhpAttributes(json)) + + private def readSwitch(json: Value): PhpSwitchStmt = + val condition = readExpr(json("cond")) + val cases = json("cases").arr.map(readCase).toList + + PhpSwitchStmt(condition, cases, PhpAttributes(json)) + + private def readTry(json: Value): PhpTryStmt = + val stmts = json("stmts").arr.map(readStmt).toList + val catches = json("catches").arr.map(readCatch).toList + val finallyStmt = Option.unless(json("finally").isNull)(readFinally(json("finally"))) + + PhpTryStmt(stmts, catches, finallyStmt, PhpAttributes(json)) + + private def readThrow(json: Value): PhpThrowExpr = + val expr = readExpr(json("expr")) + + PhpThrowExpr(expr, PhpAttributes(json)) + + private def readList(json: Value): PhpListExpr = + val items = + json("items").arr.map(item => Option.unless(item.isNull)(readArrayItem(item))).toList + + PhpListExpr(items, PhpAttributes(json)) + + private def readNew(json: Value): PhpNewExpr = + val classNode = + if json("class")("nodeType").strOpt.contains("Stmt_Class") then + readClassLike(json("class"), ClassLikeTypes.Class) + else + readNameOrExpr(json, "class") + + val args = json("args").arr.map(readCallArg).toList + + PhpNewExpr(classNode, args, PhpAttributes(json)) + + private def readInclude(json: Value): PhpIncludeExpr = + val expr = readExpr(json("expr")) + val includeType = json("type").num.toInt match + case 1 => PhpIncludeType.Include + case 2 => PhpIncludeType.IncludeOnce + case 3 => PhpIncludeType.Require + case 4 => PhpIncludeType.RequireOnce + case other => + logger.warn(s"Unhandled include type: $other. Defaulting to regular include.") + PhpIncludeType.Include + + PhpIncludeExpr(expr, includeType, PhpAttributes(json)) + + private def readMatch(json: Value): PhpMatchExpr = + val condition = readExpr(json("cond")) + val matchArms = json("arms").arr.map(readMatchArm).toList + + PhpMatchExpr(condition, matchArms, PhpAttributes(json)) + + private def readMatchArm(json: Value): PhpMatchArm = + val conditions = json("conds") match + case ujson.Null => Nil + case conds => conds.arr.map(readExpr).toList + + val isDefault = json("conds").isNull + val body = readExpr(json("body")) + + PhpMatchArm(conditions, body, isDefault, PhpAttributes(json)) + + private def readYield(json: Value): PhpYieldExpr = + val key = Option.unless(json("key").isNull)(readExpr(json("key"))) + val value = Option.unless(json("value").isNull)(readExpr(json("value"))) + + PhpYieldExpr(key, value, PhpAttributes(json)) + + private def readYieldFrom(json: Value): PhpYieldFromExpr = + val expr = readExpr(json("expr")) + + PhpYieldFromExpr(expr, PhpAttributes(json)) + + private def readClosure(json: Value): PhpClosureExpr = + val params = json("params").arr.map(readParam).toList + val stmts = json("stmts").arr.map(readStmt).toList + val returnType = Option.unless(json("returnType").isNull)(readType(json("returnType"))) + val uses = json("uses").arr.map(readClosureUse).toList + val isStatic = json("static").bool + val isByRef = json("byRef").bool + val isArrowFunc = false + + PhpClosureExpr( + params, + stmts, + returnType, + uses, + isStatic, + isByRef, + isArrowFunc, + PhpAttributes(json) + ) + end readClosure + + private def readClosureUse(json: Value): PhpClosureUse = + val variable = readVariable(json("var")) + val isByRef = json("byRef").bool + + PhpClosureUse(variable, isByRef, PhpAttributes(json)) + + private def readClassConstFetch(json: Value): PhpClassConstFetchExpr = + val classNameType = json("class")("nodeType").str + val className = + if classNameType.startsWith("Name") then + readName(json("class")) + else + readExpr(json("class")) + + val constantName = json("name") match + case str: Str => Some(PhpNameExpr(str.value, PhpAttributes(json))) + case obj: Obj if obj("nodeType").strOpt.contains("Expr_Error") => None + case obj: Obj => Some(readName(obj)) + case other => throw new NotImplementedError( + s"unexpected constant name '$other' of type ${other.getClass}" + ) + + PhpClassConstFetchExpr(className, constantName, PhpAttributes(json)) + + private def readConstFetch(json: Value): PhpConstFetchExpr = + val name = readName(json("name")) + + PhpConstFetchExpr(name, PhpAttributes(json)) + + private def readArray(json: Value): PhpArrayExpr = + val items = json("items").arr.map { item => + Option.unless(item.isNull)(readArrayItem(item)) + }.toList + PhpArrayExpr(items, PhpAttributes(json)) + + private def readArrayItem(json: Value): PhpArrayItem = + val key = Option.unless(json("key").isNull)(readExpr(json("key"))) + val value = readExpr(json("value")) + val byRef = json("byRef").bool + val unpack = json("byRef").bool + + PhpArrayItem(key, value, byRef, unpack, PhpAttributes(json)) + + private def readArrayDimFetch(json: Value): PhpArrayDimFetchExpr = + val variable = readExpr(json("var")) + val dimension = Option.unless(json("dim").isNull)(readExpr(json("dim"))) + + PhpArrayDimFetchExpr(variable, dimension, PhpAttributes(json)) + + private def readErrorSuppress(json: Value): PhpErrorSuppressExpr = + val expr = readExpr(json("expr")) + PhpErrorSuppressExpr(expr, PhpAttributes(json)) + + private def readInstanceOf(json: Value): PhpInstanceOfExpr = + val expr = readExpr(json("expr")) + val className = readNameOrExpr(json, "class") + + PhpInstanceOfExpr(expr, className, PhpAttributes(json)) + + private def readShellExec(json: Value): PhpShellExecExpr = + val parts = readEncapsed(json) + + PhpShellExecExpr(parts, PhpAttributes(json)) + + private def readArrowFunction(json: Value): PhpClosureExpr = + val params = json("params").arr.map(readParam).toList + val expr = readExpr(json("expr")) + val returnType = Option.unless(json("returnType").isNull)(readType(json("returnType"))) + val isStatic = json("static").bool + val returnByRef = json("byRef").bool + val uses = Nil // Not defined for arrow shorthand + val isArrowFunc = true + + // Introduce a return here to keep arrow functions consistent with regular closures while allowing easy code re-use. + val syntheticReturn = PhpReturnStmt(Some(expr), expr.attributes) + PhpClosureExpr( + params, + syntheticReturn :: Nil, + returnType, + uses, + isStatic, + returnByRef, + isArrowFunc, + PhpAttributes(json) + ) + end readArrowFunction + + private def readPropertyFetch( + json: Value, + isNullsafe: Boolean = false, + isStatic: Boolean = false + ): PhpPropertyFetchExpr = + val expr = + if json.obj.contains("var") then + readExpr(json("var")) + else + readNameOrExpr(json, "class") + + val name = readNameOrExpr(json, "name") + + PhpPropertyFetchExpr(expr, name, isNullsafe, isStatic, PhpAttributes(json)) + + private def readReturn(json: Value): PhpReturnStmt = + val expr = Option.unless(json("expr").isNull)(readExpr(json("expr"))) + + PhpReturnStmt(expr, PhpAttributes(json)) + + private def extendsForClassLike(json: Value): List[PhpNameExpr] = + json.obj + .get("extends") + .map { + case ujson.Null => Nil + case arr: ujson.Arr => arr.arr.map(readName).toList + case obj: ujson.Obj => readName(obj) :: Nil + case other => throw new NotImplementedError( + s"unexpected 'extends' entry '$other' of type ${other.getClass}" + ) + } + .getOrElse(Nil) + + private def readClassLike(json: Value, classLikeType: String): PhpClassLikeStmt = + val name = Option.unless(json("name").isNull)(readName(json("name"))) + val modifiers = PhpModifiers.getModifierSet(json) + + val extendsNames = extendsForClassLike(json) + + val implements = json.obj.get("implements").map(_.arr.toList).getOrElse(Nil).map(readName) + val stmts = json("stmts").arr.map(readStmt).toList + + val scalarType = + json.obj.get("scalarType").flatMap(typ => Option.unless(typ.isNull)(readName(typ))) + + val hasConstructor = classLikeType == ClassLikeTypes.Class + + val attributes = PhpAttributes(json) + + PhpClassLikeStmt( + name, + modifiers, + extendsNames, + implements, + stmts, + classLikeType, + scalarType, + hasConstructor, + attributes + ) + end readClassLike + + private def readEnumCase(json: Value): PhpEnumCaseStmt = + val name = readName(json("name")) + val expr = Option.unless(json("expr").isNull)(readExpr(json("expr"))) + + PhpEnumCaseStmt(name, expr, PhpAttributes(json)) + + private def readCatch(json: Value): PhpCatchStmt = + val types = json("types").arr.map(readName).toList + val variable = Option.unless(json("var").isNull)(readExpr(json("var"))) + val stmts = json("stmts").arr.map(readStmt).toList + + PhpCatchStmt(types, variable, stmts, PhpAttributes(json)) + + private def readFinally(json: Value): PhpFinallyStmt = + val stmts = json("stmts").arr.map(readStmt).toList + + PhpFinallyStmt(stmts, PhpAttributes(json)) + + private def readCase(json: Value): PhpCaseStmt = + val condition = Option.unless(json("cond").isNull)(readExpr(json("cond"))) + val stmts = json("stmts").arr.map(readStmt).toList + + PhpCaseStmt(condition, stmts, PhpAttributes(json)) + + private def readElseIf(json: Value): PhpElseIfStmt = + val condition = readExpr(json("cond")) + val stmts = json("stmts").arr.map(readStmt).toList + + PhpElseIfStmt(condition, stmts, PhpAttributes(json)) + + private def readElse(json: Value): PhpElseStmt = + val stmts = json("stmts").arr.map(readStmt).toList + + PhpElseStmt(stmts, PhpAttributes(json)) + + private def readEncapsed(json: Value): PhpEncapsed = + PhpEncapsed(json("parts").arr.map(readExpr).toSeq, PhpAttributes(json)) + + private def readMagicConst(json: Value): PhpConstFetchExpr = + val name = json("nodeType").str match + case "Scalar_MagicConst_Class" => "__CLASS__" + case "Scalar_MagicConst_Dir" => "__DIR__" + case "Scalar_MagicConst_File" => "__FILE__" + case "Scalar_MagicConst_Function" => "__FUNCTION__" + case "Scalar_MagicConst_Line" => "__LINE__" + case "Scalar_MagicConst_Method" => "__METHOD__" + case "Scalar_MagicConst_Namespace" => "__NAMESPACE__" + case "Scalar_MagicConst_Trait" => "__TRAIT__" + + val attributes = PhpAttributes(json) + + PhpConstFetchExpr(PhpNameExpr(name, attributes), attributes) + + private def readExpr(json: Value): PhpExpr = + json("nodeType").str match + case "Scalar_String" => readString(json) + case "Scalar_DNumber" => PhpFloat(json("value").toString, PhpAttributes(json)) + case "Scalar_LNumber" => PhpInt(json("value").toString, PhpAttributes(json)) + case "Scalar_Encapsed" => readEncapsed(json) + case "Scalar_InterpolatedString" => readEncapsed(json) + case "Scalar_EncapsedStringPart" => readString(json) + case "InterpolatedStringPart" => readString(json) + + case typ if typ.startsWith("Scalar_MagicConst") => readMagicConst(json) + + case "Expr_FuncCall" => readCall(json) + case "Expr_MethodCall" => readCall(json) + case "Expr_NullsafeMethodCall" => readCall(json) + case "Expr_StaticCall" => readCall(json) + + case "Expr_Clone" => readClone(json) + case "Expr_Empty" => readEmpty(json) + case "Expr_Eval" => readEval(json) + case "Expr_Exit" => readExit(json) + case "Expr_Variable" => readVariable(json) + case "Expr_Isset" => readIsset(json) + case "Expr_Print" => readPrint(json) + case "Expr_Ternary" => readTernaryOp(json) + case "Expr_Throw" => readThrow(json) + case "Expr_List" => readList(json) + case "Expr_New" => readNew(json) + case "Expr_Include" => readInclude(json) + case "Expr_Match" => readMatch(json) + case "Expr_Yield" => readYield(json) + case "Expr_YieldFrom" => readYieldFrom(json) + case "Expr_Closure" => readClosure(json) + + case "Expr_ClassConstFetch" => readClassConstFetch(json) + case "Expr_ConstFetch" => readConstFetch(json) + + case "Expr_Array" => readArray(json) + case "Expr_ArrayDimFetch" => readArrayDimFetch(json) + case "Expr_ErrorSuppress" => readErrorSuppress(json) + case "Expr_Instanceof" => readInstanceOf(json) + case "Expr_ShellExec" => readShellExec(json) + case "Expr_ArrowFunction" => readArrowFunction(json) + + case "Expr_PropertyFetch" => readPropertyFetch(json) + case "Expr_NullsafePropertyFetch" => readPropertyFetch(json, isNullsafe = true) + case "Expr_StaticPropertyFetch" => readPropertyFetch(json, isStatic = true) + + case typ if isUnaryOpType(typ) => readUnaryOp(json) + case typ if isBinaryOpType(typ) => readBinaryOp(json) + case typ if isAssignType(typ) => readAssign(json) + case typ if isCastType(typ) => readCast(json) + + case unhandled => + logger.error(s"Found unhandled expr type: $unhandled") + ??? + + private def readClone(json: Value): PhpCloneExpr = + val expr = readExpr(json("expr")) + PhpCloneExpr(expr, PhpAttributes(json)) + + private def readEmpty(json: Value): PhpEmptyExpr = + val expr = readExpr(json("expr")) + PhpEmptyExpr(expr, PhpAttributes(json)) + + private def readEval(json: Value): PhpEvalExpr = + val expr = readExpr(json("expr")) + PhpEvalExpr(expr, PhpAttributes(json)) + + private def readExit(json: Value): PhpExitExpr = + val expr = Option.unless(json("expr").isNull)(readExpr(json("expr"))) + PhpExitExpr(expr, PhpAttributes(json)) + + private def readVariable(json: Value): PhpVariable = + if !json.obj.contains("name") then + logger.error(s"Variable did not contain name: $json") + val varAttrs = PhpAttributes(json) + val name = json("name") match + case Str(value) => readName(value).copy(attributes = varAttrs) + case Obj(_) => readNameOrExpr(json, "name") + case value => readExpr(value) + PhpVariable(name, varAttrs) + + private def readIsset(json: Value): PhpIsset = + val vars = json("vars").arr.map(readExpr).toList + PhpIsset(vars, PhpAttributes(json)) + + private def readPrint(json: Value): PhpPrint = + val expr = readExpr(json("expr")) + PhpPrint(expr, PhpAttributes(json)) + + private def readTernaryOp(json: Value): PhpTernaryOp = + val condition = readExpr(json("cond")) + val maybeThenExpr = Option.unless(json("if").isNull)(readExpr(json("if"))) + val elseExpr = readExpr(json("else")) + + PhpTernaryOp(condition, maybeThenExpr, elseExpr, PhpAttributes(json)) + + private def readNameOrExpr(json: Value, fieldName: String): PhpExpr = + val field = json(fieldName) + if field("nodeType").str.startsWith("Name") then + readName(field) + else if field("nodeType").str == "Identifier" then + readName(field) + else if field("nodeType").str == "VarLikeIdentifier" then + readVariable(field) + else + readExpr(field) + + private def readCall(json: Value): PhpCallExpr = + val jsonMap = json.obj + val nodeType = json("nodeType").str + val args = json("args").arr.map(readCallArg).toSeq + + val target = + jsonMap.get("var").map(readExpr).orElse(jsonMap.get("class").map(_ => + readNameOrExpr(jsonMap, "class") + )) + + val methodName = readNameOrExpr(json, "name") + + val isNullSafe = nodeType == "Expr_NullsafeMethodCall" + val isStatic = nodeType == "Expr_StaticCall" + + PhpCallExpr(target, methodName, args, isNullSafe, isStatic, PhpAttributes(json)) + + private def readFunction(json: Value): PhpMethodDecl = + val returnByRef = json("byRef").bool + val name = readName(json("name")) + val params = json("params").arr.map(readParam).toList + val returnType = Option.unless(json("returnType").isNull)(readType(json("returnType"))) + val stmts = json("stmts").arr.map(readStmt).toList + // Only class methods have modifiers + val modifiers = Nil + val namespacedName = + Option.unless(json("namespacedName").isNull)(readName(json("namespacedName"))) + val isClassMethod = false + + PhpMethodDecl( + name, + params, + modifiers, + returnType, + stmts, + returnByRef, + namespacedName, + isClassMethod, + PhpAttributes(json) + ) + end readFunction + + private def readClassMethod(json: Value): PhpMethodDecl = + val modifiers = PhpModifiers.getModifierSet(json) + val returnByRef = json("byRef").bool + val name = readName(json("name")) + val params = json("params").arr.map(readParam).toList + val returnType = Option.unless(json("returnType").isNull)(readType(json("returnType"))) + val stmts = + if json("stmts").isNull then + Nil + else + json("stmts").arr.map(readStmt).toList + + val namespacedName = None // only defined for functions + val isClassMethod = true + + PhpMethodDecl( + name, + params, + modifiers, + returnType, + stmts, + returnByRef, + namespacedName, + isClassMethod, + PhpAttributes(json) + ) + end readClassMethod + + private def readProperty(json: Value): PhpPropertyStmt = + val modifiers = PhpModifiers.getModifierSet(json) + val variables = json("props").arr.map(readPropertyValue).toList + val typeName = Option.unless(json("type").isNull)(readType(json("type"))) + + PhpPropertyStmt(modifiers, variables, typeName, PhpAttributes(json)) + + private def readPropertyValue(json: Value): PhpPropertyValue = + val name = readName(json("name")) + val defaultValue = Option.unless(json("default").isNull)(readExpr(json("default"))) + + PhpPropertyValue(name, defaultValue, PhpAttributes(json)) + + private def readConst(json: Value): PhpConstStmt = + val modifiers = PhpModifiers.getModifierSet(json) + + val constDeclarations = json("consts").arr.map(readConstDeclaration).toList + + PhpConstStmt(modifiers, constDeclarations, PhpAttributes(json)) + + private def readGoto(json: Value): PhpGotoStmt = + val name = readName(json("name")) + PhpGotoStmt(name, PhpAttributes(json)) + + private def readLabel(json: Value): PhpLabelStmt = + val name = readName(json("name")) + PhpLabelStmt(name, PhpAttributes(json)) + + private def readHaltCompiler(json: Value): PhpHaltCompilerStmt = + // Ignore the remaining text here since it can get quite large (common use case is to separate code from data blob) + PhpHaltCompilerStmt(PhpAttributes(json)) + + private def readNamespace(json: Value): PhpNamespaceStmt = + val name = Option.unless(json("name").isNull)(readName(json("name"))) + + val stmts = json("stmts") match + case ujson.Null => Nil + case stmts: Arr => stmts.arr.map(readStmt).toList + case unhandled => + logger.warn(s"Unhandled namespace stmts type $unhandled") + ??? + + PhpNamespaceStmt(name, stmts, PhpAttributes(json)) + + private def readDeclare(json: Value): PhpDeclareStmt = + val declares = json("declares").arr.map(readDeclareItem).toList + val stmts = Option.unless(json("stmts").isNull)(json("stmts").arr.map(readStmt).toList) + + PhpDeclareStmt(declares, stmts, PhpAttributes(json)) + + private def readUnset(json: Value): PhpUnsetStmt = + val vars = json("vars").arr.map(readExpr).toList + + PhpUnsetStmt(vars, PhpAttributes(json)) + + private def readStatic(json: Value): PhpStaticStmt = + val vars = json("vars").arr.map(readStaticVar).toList + + PhpStaticStmt(vars, PhpAttributes(json)) + + private def readGlobal(json: Value): PhpGlobalStmt = + val vars = json("vars").arr.map(readExpr).toList + + PhpGlobalStmt(vars, PhpAttributes(json)) + + private def readUse(json: Value): PhpUseStmt = + val useType = getUseType(json("type").num.toInt) + val uses = json("uses").arr.map(readUseUse(_, useType)).toList + + PhpUseStmt(uses, useType, PhpAttributes(json)) + + private def readGroupUse(json: Value): PhpGroupUseStmt = + val prefix = readName(json("prefix")) + val useType = getUseType(json("type").num.toInt) + val uses = json("uses").arr.map(readUseUse(_, useType)).toList + + PhpGroupUseStmt(prefix, uses, useType, PhpAttributes(json)) + + private def readForeach(json: Value): PhpForeachStmt = + val iterExpr = readExpr(json("expr")) + val keyVar = Option.unless(json("keyVar").isNull)(readExpr(json("keyVar"))) + val valueVar = readExpr(json("valueVar")) + val assignByRef = json("byRef").bool + val stmts = json("stmts").arr.map(readStmt).toList + + PhpForeachStmt(iterExpr, keyVar, valueVar, assignByRef, stmts, PhpAttributes(json)) + + private def readTraitUse(json: Value): PhpTraitUseStmt = + val traits = json("traits").arr.map(readName).toList + val adaptations = json("adaptations").arr.map(readTraitUseAdaptation).toList + PhpTraitUseStmt(traits, adaptations, PhpAttributes(json)) + + private def readTraitUseAdaptation(json: Value): PhpTraitUseAdaptation = + json("nodeType").str match + case "Stmt_TraitUseAdaptation_Alias" => readAliasAdaptation(json) + case "Stmt_TraitUseAdaptation_Precedence" => readPrecedenceAdaptation(json) + + private def readAliasAdaptation(json: Value): PhpAliasAdaptation = + val traitName = Option.unless(json("trait").isNull)(readName(json("trait"))) + val methodName = readName(json("method")) + val newName = Option.unless(json("newName").isNull)(readName(json("newName"))) + + val newModifier = json("newModifier") match + case ujson.Null => None + case _ => PhpModifiers.getModifierSet(json, "newModifier").headOption + PhpAliasAdaptation(traitName, methodName, newModifier, newName, PhpAttributes(json)) + + private def readPrecedenceAdaptation(json: Value): PhpPrecedenceAdaptation = + val traitName = readName(json("trait")) + val methodName = readName(json("method")) + val insteadOf = json("insteadof").arr.map(readName).toList + + PhpPrecedenceAdaptation(traitName, methodName, insteadOf, PhpAttributes(json)) + + private def readUseUse(json: Value, parentType: PhpUseType): PhpUseUse = + val name = readName(json("name")) + val alias = Option.unless(json("alias").isNull)(readName(json("alias"))) + val useType = + if parentType == PhpUseType.Unknown then + getUseType(json("type").num.toInt) + else + parentType + + PhpUseUse(name, alias, useType, PhpAttributes(json)) + + private def readStaticVar(json: Value): PhpStaticVar = + val variable = readVariable(json("var")) + val defaultValue = Option.unless(json("default").isNull)(readExpr(json("default"))) + + PhpStaticVar(variable, defaultValue, PhpAttributes(json)) + + private def readDeclareItem(json: Value): PhpDeclareItem = + val key = readName(json("key")) + val value = readExpr(json("value")) + + PhpDeclareItem(key, value, PhpAttributes(json)) + + private def readConstDeclaration(json: Value): PhpConstDeclaration = + val name = readName(json("name")) + val value = readExpr(json("value")) + val namespacedName = + Option.unless(json("namespacedName").isNull)(readName(json("namespacedName"))) + + PhpConstDeclaration(name, value, namespacedName, PhpAttributes(json)) + + private def readParam(json: Value): PhpParam = + val paramType = Option.unless(json("type").isNull)(readType(json("type"))) + PhpParam( + name = json("var")("name").str, + paramType = paramType, + byRef = json("byRef").bool, + isVariadic = json("variadic").bool, + default = json.obj.get("default").filterNot(_.isNull).map(readExpr), + flags = json("flags").num.toInt, + attributes = PhpAttributes(json) + ) + + private def readName(json: Value): PhpNameExpr = + json match + case Str(name) => PhpNameExpr(name, PhpAttributes.Empty) + + case Obj(value) if value.get("nodeType").map(_.str).contains("Name_FullyQualified") => + val name = value("parts").arr.map(_.str).mkString(NamespaceDelimiter) + PhpNameExpr(name, PhpAttributes(json)) + + case Obj(value) if value.get("nodeType").map(_.str).contains("Name") => + // TODO Can this case just be merged with Name_FullyQualified? + val name = value("parts").arr.map(_.str).mkString(NamespaceDelimiter) + PhpNameExpr(name, PhpAttributes(json)) + + case Obj(value) if value.get("nodeType").map(_.str).contains("Identifier") => + val name = value("name").str + PhpNameExpr(name, PhpAttributes(json)) + + case Obj(value) if value.get("nodeType").map(_.str).contains("VarLikeIdentifier") => + val name = value("name").str + PhpNameExpr(name, PhpAttributes(json)) + + case unhandled => + logger.error(s"Found unhandled name type $unhandled: $json") + ??? // TODO: other matches are possible? + + /** One of Identifier, Name, or Complex Type (Nullable, Intersection, or Union) + */ + private def readType(json: Value): PhpNameExpr = + json match + case Obj(value) if value.get("nodeType").map(_.str).contains("NullableType") => + val containedName = readType(value("type")).name + PhpNameExpr(s"?$containedName", attributes = PhpAttributes(json)) + + case Obj(value) if value.get("nodeType").map(_.str).contains("IntersectionType") => + val names = value("types").arr.map(readName).map(_.name) + PhpNameExpr(names.mkString("&"), PhpAttributes(json)) + + case Obj(value) if value.get("nodeType").map(_.str).contains("UnionType") => + val names = value("types").arr.map(readType).map(_.name) + PhpNameExpr(names.mkString("|"), PhpAttributes(json)) + + case other => readName(other) + + private def readUnaryOp(json: Value): PhpUnaryOp = + val opType = UnaryOpTypeMap(json("nodeType").str) + + val expr = + if json.obj.contains("expr") then + readExpr(json.obj("expr")) + else if json.obj.contains("var") then + readExpr(json.obj("var")) + else + throw new UnsupportedOperationException( + s"Expected expr or var field in unary op but found $json" + ) + + PhpUnaryOp(opType, expr, PhpAttributes(json)) + + private def readBinaryOp(json: Value): PhpBinaryOp = + val opType = BinaryOpTypeMap(json("nodeType").str) + + val leftExpr = readExpr(json("left")) + val rightExpr = readExpr(json("right")) + + PhpBinaryOp(opType, leftExpr, rightExpr, PhpAttributes(json)) + + private def readAssign(json: Value): PhpAssignment = + val nodeType = json("nodeType").str + val opType = AssignTypeMap(nodeType) + + val target = readExpr(json("var")) + val source = readExpr(json("expr")) + + val isRefAssign = nodeType == "Expr_AssignRef" + + PhpAssignment(opType, target, source, isRefAssign, PhpAttributes(json)) + + private def readCast(json: Value): PhpCast = + val typ = CastTypeMap(json("nodeType").str) + val expr = readExpr(json("expr")) + + PhpCast(typ, expr, PhpAttributes(json)) + + private def readCallArg(json: Value): PhpArgument = + json("nodeType").str match + case "Arg" => + PhpArg( + expr = readExpr(json("value")), + parameterName = json.obj.get("name").filterNot(_.isNull).map(_("name").str), + byRef = json("byRef").bool, + unpack = json("unpack").bool, + attributes = PhpAttributes(json) + ) + + case "VariadicPlaceholder" => PhpVariadicPlaceholder(PhpAttributes(json)) + + def fromJson(jsonInput: Value): PhpFile = + readFile(jsonInput) +end Domain diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/PhpParser.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/PhpParser.scala new file mode 100644 index 00000000..b982e21b --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/parser/PhpParser.scala @@ -0,0 +1,148 @@ +package io.appthreat.php2atom.parser + +import better.files.File +import io.appthreat.php2atom.Config +import io.appthreat.php2atom.parser.Domain.PhpFile +import io.appthreat.x2cpg.utils.ExternalCommand +import org.slf4j.LoggerFactory + +import java.nio.file.Paths +import scala.io.Source +import scala.util.{Failure, Success, Try} + +class PhpParser private (phpParserPath: String, phpIniPath: String): + + private val logger = LoggerFactory.getLogger(this.getClass) + + private def phpParseCommand(filename: String): String = + val phpParserCommands = "--with-recovery --resolve-names --json-dump" + s"php --php-ini $phpIniPath $phpParserPath $phpParserCommands $filename" + + def parseFile(inputPath: String, phpIniOverride: Option[String]): Option[PhpFile] = + val inputFile = File(inputPath) + val inputFilePath = inputFile.canonicalPath + val inputDirectory = inputFile.parent.canonicalPath + + val command = phpParseCommand(inputFilePath) + + ExternalCommand.runMultiple(command, inputDirectory) match + case Success(output) => + processParserOutput(output, inputFilePath) + + case Failure(exception) => + logger.error(s"Failure running php-parser with $command", exception.getMessage()) + None + + private def processParserOutput(output: String, filename: String): Option[PhpFile] = + val maybeJson = + linesToJsonValue(output.split(System.lineSeparator()).toIndexedSeq, filename) + + maybeJson.flatMap(jsonValueToPhpFile(_, filename)) + + private def linesToJsonValue(lines: Seq[String], filename: String): Option[ujson.Value] = + if lines.exists(_.startsWith("[")) then + val jsonString = lines.dropWhile(_.charAt(0) != '[').mkString("\n") + Try(Option(ujson.read(jsonString))) match + case Success(Some(value)) => Some(value) + + case Success(None) => + logger.error(s"Parsing json string for $filename resulted in null return value") + None + + case Failure(exception) => + logger.error( + s"Parsing json string for $filename failed with exception", + exception + ) + None + else + logger.warn(s"No JSON output for $filename") + None + + private def jsonValueToPhpFile(json: ujson.Value, filename: String): Option[PhpFile] = + Try(Domain.fromJson(json)) match + case Success(phpFile) => Some(phpFile) + + case Failure(e) => + logger.error(s"Failed to generate intermediate AST for $filename", e) + None +end PhpParser + +object PhpParser: + private val logger = LoggerFactory.getLogger(this.getClass()) + + val PhpParserBinEnvVar = "PHP_PARSER_BIN" + + private def defaultPhpIni: String = + val iniContents = Source.fromResource("php.ini").getLines().mkString(System.lineSeparator()) + + val tmpIni = File.newTemporaryFile(suffix = "-php.ini").deleteOnExit() + tmpIni.writeText(iniContents) + tmpIni.canonicalPath + + private def isPhpAstgenSupported: Boolean = + val result = ExternalCommand.run("phpastgen --help", ".") + result match + case Success(listString) => + true + case Failure(exception) => + false + + private def defaultPhpParserBin: String = + if isPhpAstgenSupported then + "phpastgen" + else + val dir = + Paths.get( + this.getClass.getProtectionDomain.getCodeSource.getLocation.toURI + ).toAbsolutePath.toString + + val fixedDir = new java.io.File(dir.substring(0, dir.indexOf("php2atom"))).toString + + Paths.get( + fixedDir, + "php2atom", + "vendor", + "bin", + "php-parse" + ).toAbsolutePath.toString + + private def configOverrideOrDefaultPath( + identifier: String, + maybeOverride: Option[String], + defaultValue: => String + ): Option[String] = + val pathString = maybeOverride match + case Some(overridePath) if overridePath.nonEmpty => + logger.debug(s"Using override path for $identifier: $overridePath") + overridePath + + case _ => + logger.debug(s"$identifier path not overridden. Using default: $defaultValue") + defaultValue + + File(pathString) match + case file if file.exists() && file.isRegularFile() => Some(file.canonicalPath) + + case _ => + logger.error(s"Invalid path for $identifier: $pathString") + None + end configOverrideOrDefaultPath + + private def maybePhpParserPath(config: Config): Option[String] = + val phpParserPathOverride = + config.phpParserBin + .orElse(Option(System.getenv(PhpParserBinEnvVar))) + + configOverrideOrDefaultPath("PhpParserBin", phpParserPathOverride, defaultPhpParserBin) + + private def maybePhpIniPath(config: Config): Option[String] = + configOverrideOrDefaultPath("PhpIni", config.phpIni, defaultPhpIni) + + def getParser(config: Config): Option[PhpParser] = + for ( + phpParserPath <- maybePhpParserPath(config); + phpIniPath <- maybePhpIniPath(config) + ) + yield new PhpParser(phpParserPath, phpIniPath) +end PhpParser diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AnyTypePass.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AnyTypePass.scala new file mode 100644 index 00000000..48138e7b --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AnyTypePass.scala @@ -0,0 +1,21 @@ +package io.appthreat.php2atom.passes + +import io.appthreat.php2atom.astcreation.AstCreator +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.nodes.AstNode +import io.shiftleft.codepropertygraph.generated.nodes.Call.PropertyDefaults +import io.shiftleft.passes.ConcurrentWriterCpgPass +import io.shiftleft.semanticcpg.language.* + +// TODO This is a hack for a customer issue. Either extend this to handle type full names properly, +// or do it elsewhere. +class AnyTypePass(cpg: Cpg) extends ConcurrentWriterCpgPass[AstNode](cpg): + + override def generateParts(): Array[AstNode] = + cpg.has(PropertyNames.TYPE_FULL_NAME, PropertyDefaults.TypeFullName).collectAll[ + AstNode + ].toArray + + override def runOnPart(diffGraph: DiffGraphBuilder, node: AstNode): Unit = + diffGraph.setNodeProperty(node, PropertyNames.TYPE_FULL_NAME, AstCreator.TypeConstants.Any) diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AstCreationPass.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AstCreationPass.scala new file mode 100644 index 00000000..1ffa6f38 --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AstCreationPass.scala @@ -0,0 +1,43 @@ +package io.appthreat.php2atom.passes + +import better.files.File +import io.appthreat.php2atom.Config +import io.appthreat.php2atom.astcreation.AstCreator +import io.appthreat.php2atom.parser.PhpParser +import io.appthreat.x2cpg.datastructures.Global +import io.appthreat.x2cpg.{SourceFiles, ValidationMode} +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.passes.ConcurrentWriterCpgPass +import org.slf4j.LoggerFactory + +import scala.jdk.CollectionConverters.* + +class AstCreationPass(config: Config, cpg: Cpg, parser: PhpParser)(implicit + withSchemaValidation: ValidationMode +) extends ConcurrentWriterCpgPass[String](cpg): + + private val logger = LoggerFactory.getLogger(this.getClass) + + val PhpSourceFileExtensions: Set[String] = Set(".php") + + override def generateParts(): Array[String] = SourceFiles + .determine( + config.inputPath, + PhpSourceFileExtensions + ) + .toArray + + override def runOnPart(diffGraph: DiffGraphBuilder, filename: String): Unit = + val relativeFilename = if filename == config.inputPath then + File(filename).name + else + File(config.inputPath).relativize(File(filename)).toString + parser.parseFile(filename, config.phpIni) match + case Some(parseResult) => + diffGraph.absorb( + new AstCreator(relativeFilename, parseResult)(config.schemaValidation).createAst() + ) + + case None => + logger.warn(s"Could not parse file $filename. Results will be missing!") +end AstCreationPass diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AstParentInfoPass.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AstParentInfoPass.scala new file mode 100644 index 00000000..ff6206d8 --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/AstParentInfoPass.scala @@ -0,0 +1,36 @@ +package io.appthreat.php2atom.passes + +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, NamespaceBlock, Method, TypeDecl} +import io.shiftleft.passes.ConcurrentWriterCpgPass +import io.shiftleft.semanticcpg.language.* + +class AstParentInfoPass(cpg: Cpg) extends ConcurrentWriterCpgPass[AstNode](cpg): + + override def generateParts(): Array[AstNode] = + (cpg.method ++ cpg.typeDecl).toArray + + override def runOnPart(diffGraph: DiffGraphBuilder, node: AstNode): Unit = + findParent(node).foreach { parentNode => + val astParentType = parentNode.label + val astParentFullName = parentNode.property(PropertyNames.FULL_NAME) + + diffGraph.setNodeProperty(node, PropertyNames.AST_PARENT_TYPE, astParentType) + diffGraph.setNodeProperty(node, PropertyNames.AST_PARENT_FULL_NAME, astParentFullName) + } + + private def hasValidContainingNodes(nodes: Iterator[AstNode]): Iterator[AstNode] = + nodes.collect { + case m: Method => m + case t: TypeDecl => t + case n: NamespaceBlock => n + } + + def findParent(node: AstNode): Option[AstNode] = + node.start + .repeat(_.astParent)( + _.until(hasValidContainingNodes(_)).emit(hasValidContainingNodes(_)) + ) + .find(_ != node) +end AstParentInfoPass diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/ClosureRefPass.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/ClosureRefPass.scala new file mode 100644 index 00000000..87f94b57 --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/ClosureRefPass.scala @@ -0,0 +1,73 @@ +package io.appthreat.php2atom.passes + +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.{ClosureBinding, Method, MethodRef} +import io.shiftleft.passes.ConcurrentWriterCpgPass +import io.shiftleft.semanticcpg.language.* +import org.slf4j.LoggerFactory +import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.shiftleft.codepropertygraph.generated.nodes.AstNode +import io.shiftleft.codepropertygraph.generated.nodes.Local + +class ClosureRefPass(cpg: Cpg) extends ConcurrentWriterCpgPass[ClosureBinding](cpg): + private val logger = LoggerFactory.getLogger(this.getClass) + + override def generateParts(): Array[ClosureBinding] = cpg.all.collectAll[ClosureBinding].toArray + + /** The AstCreator adds closureBindingIds and ClosureBindings for captured locals, but does not + * add the required REF edges from the ClosureBinding to the captured node since the captured + * node may be a Local that is created by the LocalCreationPass and does not exist during AST + * creation. + * + * This pass attempts to find the captured node in the method containing the MethodRef to the + * closure method, since that is the scope in which the closure would have originally been + * created. + */ + override def runOnPart(diffGraph: DiffGraphBuilder, closureBinding: ClosureBinding): Unit = + closureBinding.captureIn.collectAll[MethodRef].toList match + case Nil => + logger.error( + s"No MethodRef corresponding to closureBinding ${closureBinding.closureBindingId}" + ) + + case methodRef :: Nil => + addRefToCapturedNode(diffGraph, closureBinding, getMethod(methodRef)) + + case methodRefs => + logger.error( + s"Mutliple MethodRefs corresponding to closureBinding ${closureBinding.closureBindingId}" + ) + logger.debug(s"${closureBinding.closureBindingId} MethodRefs = ${methodRefs}") + + private def getMethod(methodRef: MethodRef): Option[Method] = + methodRef.start.repeat(_.astParent)( + _.until(_.isMethod).emit(_.isMethod) + ).isMethod.headOption + + private def addRefToCapturedNode( + diffGraph: DiffGraphBuilder, + closureBinding: ClosureBinding, + method: Option[Method] + ): Unit = + method match + case None => + logger.warn( + s"No parent method for methodRef for ${closureBinding.closureBindingId}. REF edge will be missing" + ) + + case Some(method) => + closureBinding.closureOriginalName.foreach { name => + lazy val locals = + method.start.repeat(_.astChildren.filterNot(_.isMethod))( + _.emit(_.isLocal) + ).collectAll[Local] + val maybeCaptured = + method.parameter + .find(_.name == name) + .orElse(locals.find(_.name == name)) + + maybeCaptured.foreach { captured => + diffGraph.addEdge(closureBinding, captured, EdgeTypes.REF) + } + } +end ClosureRefPass diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/LocalCreationPass.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/LocalCreationPass.scala new file mode 100644 index 00000000..11300b93 --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/LocalCreationPass.scala @@ -0,0 +1,135 @@ +package io.appthreat.php2atom.passes + +import io.shiftleft.passes.ConcurrentWriterCpgPass +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.shiftleft.codepropertygraph.generated.nodes.{ + AstNode, + Call, + Identifier, + Method, + NamespaceBlock, + NewLocal, + NewNode, + TypeDecl +} +import io.shiftleft.semanticcpg.language.* +import io.appthreat.php2atom.astcreation.AstCreator +import io.appthreat.php2atom.parser.Domain +import io.appthreat.php2atom.parser.Domain.PhpOperators +import io.appthreat.x2cpg.AstNodeBuilder +import io.shiftleft.codepropertygraph.generated.PropertyNames + +object LocalCreationPass: + def allLocalCreationPasses(cpg: Cpg): Iterator[LocalCreationPass[? <: AstNode]] = + Iterator(new NamespaceLocalPass(cpg), new MethodLocalPass(cpg)) + +abstract class LocalCreationPass[ScopeType <: AstNode](cpg: Cpg) + extends ConcurrentWriterCpgPass[ScopeType](cpg) + with AstNodeBuilder[AstNode, LocalCreationPass[ScopeType]]: + override protected def line(node: AstNode) = node.lineNumber + override protected def column(node: AstNode) = node.columnNumber + override protected def lineEnd(node: AstNode): Option[Integer] = None + override protected def columnEnd(node: AstNode): Option[Integer] = None + override protected def code(node: AstNode): String = node.code + + protected def getIdentifiersInScope(node: AstNode): List[Identifier] = + node match + case identifier: Identifier => identifier :: Nil + case _: TypeDecl | _: Method | _: NamespaceBlock => Nil + case _ if node.astChildren.isEmpty => Nil + case call: Call if call.name == PhpOperators.declareFunc => + // TODO Handle declares properly + // but for now don't change behaviour. + Nil + case _ => node.astChildren.flatMap(getIdentifiersInScope).toList + + protected def localsForIdentifiers( + identifierMap: Map[String, List[Identifier]] + ): List[(NewLocal, List[Identifier])] = + identifierMap + .map { case identifierName -> identifiers => + val code = s"$$$identifierName" + val local = + localNode( + identifiers.head, + identifierName, + code, + AstCreator.TypeConstants.Any, + closureBindingId = None + ) + (local -> identifiers) + } + .toList + .sortBy { case (local, _) => local.name } + + protected def addRefEdges( + diffGraph: DiffGraphBuilder, + localPairs: List[(NewLocal, List[Identifier])] + ): Unit = + localPairs.foreach { case (local, identifiers) => + identifiers.foreach { identifier => + diffGraph.addEdge(identifier, local, EdgeTypes.REF) + } + } + + protected def prependLocalsToBody( + diffGraph: DiffGraphBuilder, + bodyNode: AstNode, + locals: List[NewLocal] + ): Unit = + val originalChildren = bodyNode.astChildren.l + + bodyNode.outE(EdgeTypes.AST).foreach(diffGraph.removeEdge) + + locals.zipWithIndex.foreach { case (local, idx) => + local.order(idx + 1) + } + + val localCount = locals.size + + originalChildren.foreach { node => + diffGraph.setNodeProperty(node, PropertyNames.ORDER, node.order + localCount) + } + + (locals ++ originalChildren).foreach { node => + diffGraph.addEdge(bodyNode, node, EdgeTypes.AST) + } + end prependLocalsToBody + + protected def addLocalsToAst( + diffGraph: DiffGraphBuilder, + bodyNode: AstNode, + excludeIdentifierFn: Identifier => Boolean + ): Unit = + val identifierMap = + getIdentifiersInScope(bodyNode) + .filter(_.refOut.isEmpty) + .filterNot(excludeIdentifierFn) + .groupBy(_.name) + + val localPairs = localsForIdentifiers(identifierMap) + + if localPairs.nonEmpty then + val locals = localPairs.map { case (local, _) => local } + + addRefEdges(diffGraph, localPairs) + prependLocalsToBody(diffGraph, bodyNode, locals) +end LocalCreationPass + +class NamespaceLocalPass(cpg: Cpg) extends LocalCreationPass[NamespaceBlock](cpg): + override def generateParts(): Array[NamespaceBlock] = cpg.namespaceBlock.toArray + + override def runOnPart(diffGraph: DiffGraphBuilder, namespace: NamespaceBlock): Unit = + addLocalsToAst(diffGraph, namespace, excludeIdentifierFn = _ => false) + +class MethodLocalPass(cpg: Cpg) extends LocalCreationPass[Method](cpg): + override def generateParts(): Array[Method] = cpg.method.internal.toArray + + override def runOnPart(diffGraph: DiffGraphBuilder, method: Method): Unit = + val parameters = method.parameter.name.toSet + addLocalsToAst( + diffGraph, + method.body, + excludeIdentifierFn = identifier => parameters.contains(identifier.name) + ) diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/PhpSetKnownTypes.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/PhpSetKnownTypes.scala new file mode 100644 index 00000000..ccedeeab --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/PhpSetKnownTypes.scala @@ -0,0 +1,79 @@ +package io.appthreat.php2atom.passes + +import better.files.File +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.passes.ForkJoinParallelCpgPass +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.language.operatorextension.OpNodes +import org.slf4j.{Logger, LoggerFactory} +import overflowdb.BatchedUpdate + +import scala.io.Source +import java.io.{File as JFile} + +// Corresponds to a parsed row in the known functions file +case class KnownFunction( + name: String, + // return types. A function has at most one return value, but with one or more types. + rTypes: Seq[String] = Seq.empty, + // Index 0 = parameter at P0. A function has potentially multiple parameters, each with one or more types. + pTypes: Seq[Seq[String]] = Seq.empty +) + +/** Sets the return and parameter types for builtin functions with known function signatures. + * + * TODO: Need to handle variadic arguments. + */ +class PhpSetKnownTypesPass(cpg: Cpg, knownTypesFile: Option[JFile] = None) + extends ForkJoinParallelCpgPass[KnownFunction](cpg): + + private val logger = LoggerFactory.getLogger(getClass) + + override def generateParts(): Array[KnownFunction] = + /* parse file and return each row as a KnownFunction object */ + val source = knownTypesFile match + case Some(file) => Source.fromFile(file) + case _ => Source.fromResource("known_function_signatures.txt") + val contents = source.getLines().filterNot(_.startsWith("//")) + val arr = contents.flatMap(line => createKnownFunctionFromLine(line)).toArray + source.close + arr + + override def runOnPart( + builder: overflowdb.BatchedUpdate.DiffGraphBuilder, + part: KnownFunction + ): Unit = + /* calculate the result of this part - this is done as a concurrent task */ + val builtinMethod = cpg.method.fullNameExact(part.name).l + builtinMethod.foreach(mNode => + setTypes(builder, mNode.methodReturn, part.rTypes) + (mNode.parameter.l zip part.pTypes).map((p, pTypes) => setTypes(builder, p, pTypes)) + ) + + def createKnownFunctionFromLine(line: String): Option[KnownFunction] = + line.split(";").map(_.strip).toList match + case Nil => None + case name :: Nil => Some(KnownFunction(name)) + case name :: rTypes :: Nil => Some(KnownFunction(name, scanReturnTypes(rTypes))) + case name :: rTypes :: pTypes => + Some(KnownFunction(name, scanReturnTypes(rTypes), scanParamTypes(pTypes))) + + /* From comma separated list of types, create list of types. */ + def scanReturnTypes(rTypesRaw: String): Seq[String] = rTypesRaw.split(",").map(_.strip).toSeq + + /* From a semicolon separated list of parameters, each with a comma separated list of types, + * create a list of lists of types. */ + def scanParamTypes(pTypesRawArr: List[String]): Seq[Seq[String]] = + pTypesRawArr.map(paramTypeRaw => paramTypeRaw.split(",").map(_.strip).toSeq).toSeq + + protected def setTypes( + builder: overflowdb.BatchedUpdate.DiffGraphBuilder, + n: StoredNode, + types: Seq[String] + ): Unit = + if types.size == 1 then builder.setNodeProperty(n, PropertyNames.TYPE_FULL_NAME, types.head) + else builder.setNodeProperty(n, PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, types) +end PhpSetKnownTypesPass diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/ArrayIndexTracker.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/ArrayIndexTracker.scala new file mode 100644 index 00000000..b723c71a --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/ArrayIndexTracker.scala @@ -0,0 +1,16 @@ +package io.appthreat.php2atom.datastructures + +class ArrayIndexTracker: + private var currentValue = 0 + + def next: String = + val nextVal = currentValue + currentValue += 1 + nextVal.toString + + def updateValue(newValue: Int): Unit = + if newValue >= currentValue then + currentValue = newValue + 1 + +object ArrayIndexTracker: + def apply(): ArrayIndexTracker = new ArrayIndexTracker diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/Scope.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/Scope.scala new file mode 100644 index 00000000..67d2cefe --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/Scope.scala @@ -0,0 +1,106 @@ +package io.appthreat.php2atom.utils + +import io.appthreat.php2atom.astcreation.AstCreator.NameConstants +import io.appthreat.php2atom.utils.PhpScopeElement +import io.appthreat.x2cpg.Ast +import io.appthreat.x2cpg.datastructures.{Scope as X2CpgScope} +import io.shiftleft.codepropertygraph.generated.nodes.{ + NewMethod, + NewNamespaceBlock, + NewNode, + NewTypeDecl +} +import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal +import org.slf4j.LoggerFactory + +import scala.collection.mutable + +class Scope(implicit nextClosureName: () => String) + extends X2CpgScope[String, NewNode, PhpScopeElement]: + + private val logger = LoggerFactory.getLogger(this.getClass) + + private var constAndStaticInits: List[mutable.ArrayBuffer[Ast]] = Nil + private var fieldInits: List[mutable.ArrayBuffer[Ast]] = Nil + private val anonymousMethods = mutable.ArrayBuffer[Ast]() + + def pushNewScope(scopeNode: NewNode): Unit = + scopeNode match + case method: NewMethod => + super.pushNewScope(PhpScopeElement(method)) + + case typeDecl: NewTypeDecl => + constAndStaticInits = mutable.ArrayBuffer[Ast]() :: constAndStaticInits + fieldInits = mutable.ArrayBuffer[Ast]() :: fieldInits + super.pushNewScope(PhpScopeElement(typeDecl)) + + case namespace: NewNamespaceBlock => + super.pushNewScope(PhpScopeElement(namespace)) + + case invalid => + logger.warn(s"pushNewScope called with invalid node $invalid. Ignoring!") + + override def popScope(): Option[PhpScopeElement] = + val scopeNode = super.popScope() + + scopeNode.map(_.node) match + case Some(_: NewTypeDecl) => + // TODO This is unsafe to catch errors for now + constAndStaticInits = constAndStaticInits.tail + fieldInits = fieldInits.tail + + case _ => // Nothing to do here + scopeNode + + override def addToScope(identifier: String, variable: NewNode): PhpScopeElement = + super.addToScope(identifier, variable) + + def addAnonymousMethod(methodAst: Ast): Unit = anonymousMethods.addOne(methodAst) + + def getAndClearAnonymousMethods: List[Ast] = + val methods = anonymousMethods.toList + anonymousMethods.clear() + methods + + def getEnclosingNamespaceNames: List[String] = + stack.map(_.scopeNode.node).collect { case ns: NewNamespaceBlock => ns.name }.reverse + + def getEnclosingTypeDeclTypeName: Option[String] = + stack.map(_.scopeNode.node).collectFirst { case td: NewTypeDecl => td }.map(_.name) + + def getEnclosingTypeDeclTypeFullName: Option[String] = + stack.map(_.scopeNode.node).collectFirst { case td: NewTypeDecl => td }.map(_.fullName) + + def addConstOrStaticInitToScope(ast: Ast): Unit = + addInitToScope(ast, constAndStaticInits) + def getConstAndStaticInits: List[Ast] = + getInits(constAndStaticInits) + + def addFieldInitToScope(ast: Ast): Unit = + addInitToScope(ast, fieldInits) + + def getFieldInits: List[Ast] = + getInits(fieldInits) + + def getScopedClosureName: String = + stack.headOption match + case Some(scopeElement) => + scopeElement.scopeNode.getClosureMethodName + + case None => + logger.warn( + "BUG: Attempting to get scopedClosureName, but no scope has been push. Defaulting to unscoped" + ) + NameConstants.Closure + + private def addInitToScope(ast: Ast, initList: List[mutable.ArrayBuffer[Ast]]): Unit = + // TODO This is unsafe to catch errors for now + initList.head.addOne(ast) + + private def getInits(initList: List[mutable.ArrayBuffer[Ast]]): List[Ast] = + // TODO This is unsafe to catch errors for now + val ret = initList.head.toList + // These ASTs should only be added once to avoid aliasing issues. + initList.head.clear() + ret +end Scope diff --git a/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/ScopeElement.scala b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/ScopeElement.scala new file mode 100644 index 00000000..7e7ded98 --- /dev/null +++ b/platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/utils/ScopeElement.scala @@ -0,0 +1,31 @@ +package io.appthreat.php2atom.utils + +import io.appthreat.php2atom.parser.Domain.InstanceMethodDelimiter +import io.shiftleft.codepropertygraph.generated.nodes.{ + NewMethod, + NewNamespaceBlock, + NewNode, + NewTypeDecl +} + +class PhpScopeElement private (val node: NewNode, scopeName: String)(implicit + nextClosureName: () => String +): + + def getClosureMethodName: String = + s"$scopeName$InstanceMethodDelimiter${nextClosureName()}" + +object PhpScopeElement: + def apply(method: NewMethod)(implicit nextClosureName: () => String): PhpScopeElement = + new PhpScopeElement(method, method.fullName) + + def apply(typeDecl: NewTypeDecl)(implicit nextClosureName: () => String): PhpScopeElement = + new PhpScopeElement(typeDecl, typeDecl.fullName) + + def apply(namespace: NewNamespaceBlock)(implicit + nextClosureName: () => String + ): PhpScopeElement = + new PhpScopeElement(namespace, namespace.fullName) + + def unapply(scopeElement: PhpScopeElement): Option[NewNode] = + Some(scopeElement.node) diff --git a/platform/frontends/php2atom/src/test/resources/builtin_functions.txt b/platform/frontends/php2atom/src/test/resources/builtin_functions.txt new file mode 100644 index 00000000..bc05192b --- /dev/null +++ b/platform/frontends/php2atom/src/test/resources/builtin_functions.txt @@ -0,0 +1 @@ +abs \ No newline at end of file diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/config/ConfigTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/config/ConfigTests.scala new file mode 100644 index 00000000..1d289de0 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/config/ConfigTests.scala @@ -0,0 +1,41 @@ +package io.appthreat.php2atom.config + +import io.appthreat.php2atom.Main +import io.appthreat.php2atom.Config + +import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.matchers.should.Matchers +import io.appthreat.x2cpg.X2Cpg +import org.scalatest.Inside + +class ConfigTests extends AnyWordSpec with Matchers with Inside { + + "php2cpg command line args should be parsed correctly" in { + val parser = Main.cmdLineParser + val args = Array( + // Common args + "INPUT", + "--output", + "OUTPUT", + "--exclude", + "1EXCLUDE_FILE,2EXCLUDE_FILE", + "--exclude-regex", + "EXCLUDE_REGEX", + // Frontend-specific args + "--php-ini", + "PHP_INI" + ) + + def getSuffix(s: String, n: Int): String = { + s.reverse.take(n).reverse + } + + inside(X2Cpg.parseCommandLine(args, parser, Config())) { case Some(config) => + config.inputPath.endsWith("INPUT") shouldBe true + config.outputPath shouldBe "OUTPUT" + config.ignoredFiles.map(getSuffix(_, 13)).toSet shouldBe Set("1EXCLUDE_FILE", "2EXCLUDE_FILE") + config.ignoredFilesRegex.toString shouldBe "EXCLUDE_REGEX" + config.phpIni shouldBe Some("PHP_INI") + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/dataflow/IntraMethodDataflowTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/dataflow/IntraMethodDataflowTests.scala new file mode 100644 index 00000000..c401ca25 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/dataflow/IntraMethodDataflowTests.scala @@ -0,0 +1,32 @@ +package io.appthreat.php2atom.dataflow + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.shiftleft.semanticcpg.language._ +import io.appthreat.dataflowengineoss.language._ + +class IntraMethodDataflowTests extends PhpCode2CpgFixture(runOssDataflow = true) { + "flows from parameters to corresponding identifiers should be found" in { + val cpg = code(""" new PhpCfgTestCpg) { + override def code(code: String): PhpCfgTestCpg = { + super.code(s" + indexAccess.name shouldBe Operators.indexAccess + indexAccess.code shouldBe "$array[$key]" + indexAccess.lineNumber shouldBe Some(2) + + inside(indexAccess.argument.l) { case List(array: Identifier, key: Identifier) => + array.name shouldBe "array" + array.code shouldBe "$array" + array.lineNumber shouldBe Some(2) + + key.name shouldBe "key" + key.code shouldBe "$key" + key.lineNumber shouldBe Some(2) + } + } + } + + "array accesses with literal keys should be represented as index accesses" in { + val cpg = code(" + indexAccess.name shouldBe Operators.indexAccess + indexAccess.code shouldBe "$array[0]" + indexAccess.lineNumber shouldBe Some(2) + + inside(indexAccess.argument.l) { case List(array: Identifier, key: Literal) => + array.name shouldBe "array" + array.code shouldBe "$array" + array.lineNumber shouldBe Some(2) + + key.code shouldBe "0" + key.lineNumber shouldBe Some(2) + } + } + } + + "assignments using the empty array dimension fetch syntax should be rewritten as array_push" in { + val cpg = code(""" + xsLocal.name shouldBe "xs" + xsLocal.lineNumber shouldBe Some(3) + + arrayPush.name shouldBe "array_push" + arrayPush.code shouldBe "$xs[] = $val" + } + } + + "associative array definitions should be lowered with the correct assignments" in { + val cpg = code(""" 1, + | "B" => 2 + |) + |""".stripMargin) + + inside(cpg.method.internal.body.astChildren.l) { case List(tmpLocal: Local, arrayBlock: Block) => + tmpLocal.name shouldBe "tmp0" + tmpLocal.code shouldBe "$tmp0" + + inside(arrayBlock.astChildren.l) { case List(aAssign: Call, bAssign: Call, tmpIdent: Identifier) => + aAssign.code shouldBe "$tmp0[\"A\"] = 1" + aAssign.lineNumber shouldBe Some(3) + + bAssign.code shouldBe "$tmp0[\"B\"] = 2" + bAssign.lineNumber shouldBe Some(4) + + tmpIdent.name shouldBe "tmp0" + tmpIdent.code shouldBe "$tmp0" + tmpIdent._localViaRefOut should contain(tmpLocal) + } + } + } + + "non-associative array definitions should be lowered with the correct index accesses and assignments" in { + val cpg = code(""" + tmpLocal.name shouldBe "tmp0" + tmpLocal.code shouldBe "$tmp0" + + inside(arrayBlock.astChildren.l) { case List(aAssign: Call, bAssign: Call, tmpIdent: Identifier) => + aAssign.code shouldBe "$tmp0[0] = \"A\"" + aAssign.lineNumber shouldBe Some(3) + + bAssign.code shouldBe "$tmp0[1] = \"B\"" + bAssign.lineNumber shouldBe Some(4) + + tmpIdent.name shouldBe "tmp0" + tmpIdent.code shouldBe "$tmp0" + tmpIdent._localViaRefOut should contain(tmpLocal) + } + } + } + + "arrays with int-compatible indices should have them treated as ints" in { + val cpg = code(""" "A" + |) + |""".stripMargin) + + inside(cpg.method.internal.body.astChildren.l) { case List(tmpLocal: Local, arrayBlock: Block) => + tmpLocal.name shouldBe "tmp0" + tmpLocal.code shouldBe "$tmp0" + + inside(arrayBlock.astChildren.l) { case List(assign: Call, tmpIdent: Identifier) => + assign.code shouldBe "$tmp0[2] = \"A\"" + inside(assign.argument.collectAll[Call].argument.l) { case List(array: Identifier, index: Literal) => + array.name shouldBe "tmp0" + array.code shouldBe "$tmp0" + + index.code shouldBe "2" + index.typeFullName shouldBe "int" + } + + tmpIdent.name shouldBe "tmp0" + tmpIdent.code shouldBe "$tmp0" + tmpIdent._localViaRefOut should contain(tmpLocal) + } + } + } + + "mixed associative array definitions should be represented with correct keys" in { + val cpg = code(""" "B", + | "C", + | 4 => "D", + | "E", + | "10" => "F", + | "G", + | 8 => "H", + |) + |""".stripMargin) + + inside(cpg.method.internal.body.astChildren.l) { case List(tmpLocal: Local, arrayBlock: Block) => + tmpLocal.name shouldBe "tmp0" + tmpLocal.code shouldBe "$tmp0" + + inside(arrayBlock.astChildren.l) { + case List( + aAssign: Call, + cAssign: Call, + fourAssign: Call, + eAssign: Call, + tenAssign: Call, + gAssign: Call, + eightAssign: Call, + tmpIdent: Identifier + ) => + aAssign.code shouldBe "$tmp0[\"A\"] = \"B\"" + cAssign.code shouldBe "$tmp0[0] = \"C\"" + fourAssign.code shouldBe "$tmp0[4] = \"D\"" + eAssign.code shouldBe "$tmp0[5] = \"E\"" + tenAssign.code shouldBe "$tmp0[10] = \"F\"" + gAssign.code shouldBe "$tmp0[11] = \"G\"" + eightAssign.code shouldBe "$tmp0[8] = \"H\"" + + tmpIdent.name shouldBe "tmp0" + tmpIdent.code shouldBe "$tmp0" + tmpIdent._localViaRefOut should contain(tmpLocal) + } + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CallTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CallTests.scala new file mode 100644 index 00000000..1ac7a684 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CallTests.scala @@ -0,0 +1,170 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.astcreation.AstCreator.NameConstants +import io.appthreat.php2atom.parser.Domain.PhpOperators +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.appthreat.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier} +import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.Block + +class CallTests extends PhpCode2CpgFixture { + "halt_compiler calls should be created correctly" in { + val cpg = code(" + haltCompiler.name shouldBe NameConstants.HaltCompiler + haltCompiler.methodFullName shouldBe NameConstants.HaltCompiler + haltCompiler.code shouldBe s"${NameConstants.HaltCompiler}()" + haltCompiler.lineNumber shouldBe Some(2) + haltCompiler.astChildren.size shouldBe 0 + } + } + + "inline HTML should be represented as an echo statement" in { + val cpg = code("TEST CODE PLEASE IGNORE") + + inside(cpg.call.name(".*echo").l) { case List(echoCall) => + echoCall.name shouldBe "echo" + echoCall.argument.code.l shouldBe List("\"TEST CODE PLEASE IGNORE\"") + } + } + + "function calls with simple names should be correct" in { + val cpg = code(" + fooCall.name shouldBe "foo" + fooCall.methodFullName shouldBe s"foo" + fooCall.signature shouldBe s"${Defines.UnresolvedSignature}(1)" + fooCall.receiver.isEmpty shouldBe true + fooCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + fooCall.lineNumber shouldBe Some(2) + fooCall.code shouldBe "foo($x)" + + inside(fooCall.argument.l) { case List(xArg: Identifier) => + xArg.name shouldBe "x" + xArg.code shouldBe "$x" + } + } + } + + "static method calls with simple names" should { + val cpg = code(" + fooCall.name shouldBe "foo" + fooCall.methodFullName shouldBe s"Foo::foo" + fooCall.receiver.isEmpty shouldBe true + fooCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + fooCall.lineNumber shouldBe Some(2) + fooCall.code shouldBe "Foo::foo($x)" + } + } + + "have the correct arguments" in { + inside(cpg.call.argument.l) { case List(xArg: Identifier) => + xArg.name shouldBe "x" + xArg.code shouldBe "$x" + } + } + + "have the correct child nodes" in { + inside(cpg.call.astChildren.l) { case List(arg: Identifier) => + arg.name shouldBe "x" + } + } + + "not create an identifier for the class target" in { + inside(cpg.identifier.l) { case List(xArg) => + xArg.name shouldBe "x" + } + } + } + + /* This possibly should exist in NamespaceTests.scala */ + "static method calls that refer to self" should { + val cpg = code(""" + |foo($x);") + + inside(cpg.call.l) { case List(fooCall) => + fooCall.name shouldBe "foo" + fooCall.methodFullName shouldBe """\$f->foo""" + fooCall.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH + fooCall.lineNumber shouldBe Some(2) + fooCall.code shouldBe "$f->foo($x)" + + inside(fooCall.argument.l) { case List(fRecv: Identifier, xArg: Identifier) => + fRecv.name shouldBe "f" + fRecv.code shouldBe "$f" + fRecv.lineNumber shouldBe Some(2) + xArg.name shouldBe "x" + xArg.code shouldBe "$x" + } + } + } + + "method calls with complex names should be correct" in { + val cpg = code("{$foo}($x)") + + inside(cpg.call.filter(_.name != Operators.fieldAccess).l) { case List(fooCall) => + fooCall.name shouldBe "$foo" + fooCall.methodFullName shouldBe """\$$f->$foo""" + fooCall.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH + fooCall.lineNumber shouldBe Some(2) + fooCall.code shouldBe "$$f->$foo($x)" + + inside(fooCall.argument.l) { case List(fRecv: Call, xArg: Identifier) => + fRecv.name shouldBe Operators.fieldAccess + fRecv.code shouldBe "$$f->$foo" + fRecv.lineNumber shouldBe Some(2) + + inside(fRecv.argument.l) { case List(fVar: Identifier, fooVar: Identifier) => + fVar.name shouldBe "f" + fVar.code shouldBe "$$f" + + fooVar.name shouldBe "foo" + fooVar.code shouldBe "$foo" + } + + xArg.name shouldBe "x" + xArg.code shouldBe "$x" + } + } + } + + "the code field of Call with array unpack should be correct" in { + val cpg = code(""" + | body1(), + | 'Y', 'Z' => body2(), + | }; + | sink(); + |} + |""".stripMargin) + "find that the jump targets and sink call are CFG successors of cond call" in { + inside(cpg.call.name("cond").cfgNext.l) { + case List(xTarget: JumpTarget, yTarget: JumpTarget, zTarget: JumpTarget, sink: Call) => + xTarget.name shouldBe "case \"X\"" + yTarget.name shouldBe "case \"Y\"" + zTarget.name shouldBe "case \"Z\"" + sink.code shouldBe "sink()" + } + } + + "find that the sink call is the CFG successor of the body1 call" in { + inside(cpg.call.name("body1").cfgNext.l) { case List(sinkCall: Call) => + sinkCall.code shouldBe "sink()" + } + } + + "find that the sink call is the CFG successor of the body2 call" in { + inside(cpg.call.name("body2").cfgNext.l) { case List(sinkCall: Call) => + sinkCall.code shouldBe "sink()" + } + } + } + + "the match has a default case" should { + val cpg = code(""" body1(), + | 'Y', 'Z' => body2(), + | default => body3(), + | }; + | sink(); + |} + |""".stripMargin) + + "only the jump targets are CFG successors of the cond call" in { + inside(cpg.call.name("cond").cfgNext.l) { + case List(xTarget: JumpTarget, yTarget: JumpTarget, zTarget: JumpTarget, defaultTarget: JumpTarget) => + xTarget.name shouldBe "case \"X\"" + yTarget.name shouldBe "case \"Y\"" + zTarget.name shouldBe "case \"Z\"" + defaultTarget.name shouldBe "default" + } + } + + "find that the sink call is the CFG successor of the body1 call" in { + inside(cpg.call.name("body1").cfgNext.l) { case List(sinkCall: Call) => + sinkCall.code shouldBe "sink()" + } + } + + "find that the sink call is the CFG successor of the body2 call" in { + inside(cpg.call.name("body2").cfgNext.l) { case List(sinkCall: Call) => + sinkCall.code shouldBe "sink()" + } + } + + "find that the sink call is the CFG successor of the body3 call" in { + inside(cpg.call.name("body3").cfgNext.l) { case List(sinkCall: Call) => + sinkCall.code shouldBe "sink()" + } + } + } + } + + "the CFG for if constructs" should { + val cpg = code(""" 5) { + | sink2(); + | } else { + | sink3(); + | } + | echo "foo"; + |} + |""".stripMargin) + + "find that sink1 is control dependent on the if condition" in { + inside(cpg.call.name("sink1").controlledBy.isCall.code.l) { case List(controllerCode) => + controllerCode shouldBe "$x < 10" + } + } + + "find that sink2 is control dependent on the if and elseif conditions" in { + inside(cpg.call.name("sink2").controlledBy.isCall.code.sorted.toList) { case List(xCode, yCode) => + xCode shouldBe "$x < 10" + yCode shouldBe "$y > 5" + } + } + + "find that sink3 is control dependent on the if and elseif conditions" in { + inside(cpg.call.name("sink3").controlledBy.isCall.code.sorted.toList) { case List(xCode, yCode) => + xCode shouldBe "$x < 10" + yCode shouldBe "$y > 5" + } + } + + "find that the if controls sink1" in { + inside(cpg.controlStructure.condition.codeExact("$x < 10").l) { case List(condition) => + condition.controls.isCall.name("sink1").l.size shouldBe 1 + } + } + + "find that echo post dominates all" in { + cpg.call("echo").postDominates.size shouldBe 11 + } + + "find that the method does not post dominate anything" in { + inside(cpg.method("foo").l) { case List(method) => + method.postDominates.size shouldBe 0 + } + } + } + + "the CFG for while constructs" should { + val cpg = code(""" + condition.code shouldBe "$x < 10" + } + } + + "find that the sink call does not dominate anything" in { + cpg.call.name("sink").dominates.size shouldBe 0 + } + } + + "the CFG for switch constructs" should { + val cpg = code(""" + inside(cpg.call.nameExact(s"sink$num").controlledBy.isCall.code.l) { case List(code) => + code shouldBe "$x < 10" + } + } + } + + "find that sink2 post dominates sink1" in { + cpg.call("sink2").postDominates.isCall.name.toSet should contain("sink1") + } + + "find that sink3 does not post dominate sink2" in { + cpg.call("sink3").postDominates.isCall.name.toSet should not contain "sink2" + } + } + + "the CFG for foreach constructs" should { + val cpg = code(""" s"call$num").toSet + } + + "find that call6 is post dominated by the try and finally calls" in { + cpg.call.name("call1").postDominatedBy.collectAll[Call].name.toSet shouldBe Set("call2", "call5", "call6") + } + + "find that call3 is controlled by call2" in { + cpg.call.name("call3").controlledBy.collectAll[Call].name.toSet should contain("call2") + } + + "find that call6 is not controlled by call2" in { + cpg.call.name("call6").controlledBy.collectAll[Call].name.toSet should not contain ("call2") + } + } + + "the CFG for gotos" should { + val cpg = code(""" + closureMethod + } + + closureMethod.name shouldBe "0" + closureMethod.fullName shouldBe s"0:${Defines.UnresolvedSignature}(1)" + closureMethod.code shouldBe "function 0($value)" + closureMethod.parameter.size shouldBe 1 + + inside(closureMethod.parameter.l) { case List(valueParam) => + valueParam.name shouldBe "value" + } + + inside(closureMethod.body.astChildren.l) { case List(echoCall: Call) => + echoCall.code shouldBe "echo $value" + } + } + + "have a correct MethodRef added to the AST where the closure is defined" in { + inside(cpg.assignment.argument.l) { case List(_: Identifier, methodRef: MethodRef) => + methodRef.methodFullName shouldBe s"0:${Defines.UnresolvedSignature}(1)" + methodRef.code shouldBe s"0:${Defines.UnresolvedSignature}(1)" + methodRef.lineNumber shouldBe Some(2) + } + } + } + + "long-form closures with uses " should { + val cpg = code( + """.*").l) { case List(closureMethod) => + closureMethod + } + + val expectedName = s"foo.php:->0" + closureMethod.name shouldBe expectedName + closureMethod.fullName shouldBe expectedName + closureMethod.signature shouldBe s"${Defines.UnresolvedSignature}(1)" + closureMethod.code shouldBe s"function $expectedName($$value) use($$use1, &$$use2)" + closureMethod.parameter.size shouldBe 1 + + inside(closureMethod.parameter.l) { case List(valueParam) => + valueParam.name shouldBe "value" + } + + inside(closureMethod.body.astChildren.l) { case List(use1: Local, use2: Local, echoCall: Call) => + use1.name shouldBe "use1" + use1.code shouldBe "$use1" + use1.closureBindingId shouldBe Some(s"foo.php:$expectedName:use1") + inside(cpg.all.collectAll[ClosureBinding].filter(_.closureBindingId == use1.closureBindingId).l) { + case List(closureBinding) => + closureBinding.closureOriginalName shouldBe Some("use1") + } + + use2.name shouldBe "use2" + use2.code shouldBe "&$use2" + use2.closureBindingId shouldBe Some(s"foo.php:$expectedName:use2") + + echoCall.code shouldBe "echo $value" + } + } + + "have a ref edge from the closure binding for a use to the captured node" in { + inside(cpg.all.collectAll[ClosureBinding].filter(_.closureOriginalName.contains("use1")).l) { + case List(closureBinding) => + val capturedNode = cpg.method.nameExact("").local.name("use1").head + closureBinding.refOut.toList shouldBe List(capturedNode) + } + } + + "have a correct MethodRef added to the AST where the closure is defined" in { + inside(cpg.assignment.code(".*.*").argument.l) { case List(_: Identifier, methodRef: MethodRef) => + val expectedName = s"foo.php:->0" + methodRef.methodFullName shouldBe expectedName + methodRef.code shouldBe expectedName + methodRef.lineNumber shouldBe Some(3) + } + } + } + + "arrow functions should be represented as closures with return statements" should { + val cpg = code( + """ $value + 1; + |""".stripMargin, + fileName = "foo.php" + ) + + "have the correct method AST" in { + val closureMethod = inside(cpg.method.name(".*.*").l) { case List(closureMethod) => + closureMethod + } + + val expectedName = "foo.php:->0" + closureMethod.name shouldBe expectedName + closureMethod.fullName shouldBe expectedName + closureMethod.signature shouldBe s"${Defines.UnresolvedSignature}(1)" + closureMethod.code shouldBe s"function $expectedName($$value)" + closureMethod.parameter.size shouldBe 1 + + inside(closureMethod.parameter.l) { case List(valueParam) => + valueParam.name shouldBe "value" + } + + inside(closureMethod.body.astChildren.l) { case List(methodReturn: Return) => + methodReturn.code shouldBe "return $value + 1" + } + } + + "have a correct MethodRef added to the AST where the closure is defined" in { + inside(cpg.assignment.argument.l) { case List(_: Identifier, methodRef: MethodRef) => + val expectedName = "foo.php:->0" + methodRef.methodFullName shouldBe expectedName + methodRef.code shouldBe expectedName + methodRef.lineNumber shouldBe Some(2) + } + } + } + + "multiple closures in the same file should have the correct names" in { + val cpg = code(""" $value + 1; + | $y = function ($value) { + | return $value + 2; + | } + |} + | + |class Bar { + | function bar() { + | $x = fn ($value) => $value + 1; + | $y = function ($value) { + | return $value + 2; + | } + | } + |} + |""".stripMargin) + + inside(cpg.method.name(".*.*").fullName.sorted.l) { case List(bar0, bar1, foo0, foo1) => + bar0 shouldBe "Bar->bar->2" + bar1 shouldBe "Bar->bar->3" + foo0 shouldBe "foo->0" + foo1 shouldBe "foo->1" + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CommentTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CommentTests.scala new file mode 100644 index 00000000..1ca666df --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/CommentTests.scala @@ -0,0 +1,17 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.shiftleft.semanticcpg.language._ + +class CommentTests extends PhpCode2CpgFixture { + + "parsing a file containing a Nop-wrapped comment should not result in a crash" in { + val cpg = code(""" + switchStmt.code shouldBe "switch ($cond)" + switchStmt.lineNumber shouldBe Some(2) + + inside(switchStmt.condition.l) { case List(cond: Identifier) => + cond.name shouldBe "cond" + cond.code shouldBe "$cond" + cond.lineNumber shouldBe Some(2) + } + + inside(switchStmt.whenTrue.astChildren.l) { + case List( + case0: JumpTarget, + bIdent: Identifier, + break1: ControlStructure, + case1: JumpTarget, + cIdent: Identifier, + break2: ControlStructure + ) => + case0.name shouldBe "case" + case0.code shouldBe "case 0" + case0.lineNumber shouldBe Some(3) + + bIdent.name shouldBe "b" + bIdent.code shouldBe "$b" + bIdent.lineNumber shouldBe Some(4) + + break1.controlStructureType shouldBe ControlStructureTypes.BREAK + break1.code shouldBe "break" + break1.lineNumber shouldBe Some(5) + + case1.name shouldBe "case" + case1.code shouldBe "case 1" + case1.lineNumber shouldBe Some(6) + + cIdent.name shouldBe "c" + cIdent.code shouldBe "$c" + cIdent.lineNumber shouldBe Some(7) + + break2.controlStructureType shouldBe ControlStructureTypes.BREAK + break2.code shouldBe "break" + break2.lineNumber shouldBe Some(8) + } + } + } + + "work with a default case" in { + val cpg = code(""" + switchStmt.code shouldBe "switch ($cond)" + switchStmt.lineNumber shouldBe Some(2) + + inside(switchStmt.condition.l) { case List(cond: Identifier) => + cond.name shouldBe "cond" + cond.code shouldBe "$cond" + cond.lineNumber shouldBe Some(2) + } + + inside(switchStmt.whenTrue.astChildren.l) { + case List( + case0: JumpTarget, + bIdent: Identifier, + break1: ControlStructure, + defaultCase: JumpTarget, + cIdent: Identifier + ) => + case0.name shouldBe "case" + case0.code shouldBe "case 0" + case0.lineNumber shouldBe Some(3) + + bIdent.name shouldBe "b" + bIdent.code shouldBe "$b" + bIdent.lineNumber shouldBe Some(4) + + break1.controlStructureType shouldBe ControlStructureTypes.BREAK + break1.code shouldBe "break" + break1.lineNumber shouldBe Some(5) + + defaultCase.name shouldBe "default" + defaultCase.code shouldBe "default" + defaultCase.lineNumber shouldBe Some(6) + + cIdent.name shouldBe "c" + cIdent.code shouldBe "$c" + cIdent.lineNumber shouldBe Some(7) + } + } + } + } + + "if statements" should { + "work without a body, an elseif or an else" in { + val cpg = code(" + ast.controlStructureType shouldBe ControlStructureTypes.IF + ast + } + + ifAst.code shouldBe "if ($a)" + ifAst.lineNumber shouldBe Some(2) + + inside(ifAst.condition.l) { case List(aIdent: Identifier) => + aIdent.name shouldBe "a" + aIdent.code shouldBe "$a" + aIdent.lineNumber shouldBe Some(2) + } + + inside(ifAst.astChildren.l) { case List(_, thenBlock: Block) => + thenBlock.astChildren.size shouldBe 0 + thenBlock.lineNumber shouldBe Some(2) + } + } + + "work with just a then body" in { + val cpg = code(" + ast.controlStructureType shouldBe ControlStructureTypes.IF + ast + } + + ifAst.code shouldBe "if ($a)" + ifAst.lineNumber shouldBe Some(2) + + inside(ifAst.condition.l) { case List(aIdent: Identifier) => + aIdent.name shouldBe "a" + aIdent.code shouldBe "$a" + aIdent.lineNumber shouldBe Some(2) + } + + inside(ifAst.astChildren.l) { case List(_, thenBlock: Block) => + thenBlock.lineNumber shouldBe Some(2) + + inside(thenBlock.astChildren.l) { case List(bIdentifier: Identifier) => + bIdentifier.name shouldBe "b" + bIdentifier.code shouldBe "$b" + bIdentifier.lineNumber shouldBe Some(2) + } + } + } + + "work with else" in { + val cpg = code(" + ast.controlStructureType shouldBe ControlStructureTypes.IF + ast + } + + ifAst.code shouldBe "if ($a)" + ifAst.lineNumber shouldBe Some(2) + + inside(ifAst.condition.l) { case List(aIdent: Identifier) => + aIdent.name shouldBe "a" + aIdent.code shouldBe "$a" + aIdent.lineNumber shouldBe Some(2) + } + + inside(ifAst.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + thenBlock.lineNumber shouldBe Some(2) + elseBlock.lineNumber shouldBe Some(2) + + inside(thenBlock.astChildren.l) { case List(bIdentifier: Identifier) => + bIdentifier.name shouldBe "b" + bIdentifier.code shouldBe "$b" + bIdentifier.lineNumber shouldBe Some(2) + } + + inside(elseBlock.astChildren.l) { case List(cIdentifier: Identifier) => + cIdentifier.name shouldBe "c" + cIdentifier.code shouldBe "$c" + cIdentifier.lineNumber shouldBe Some(2) + } + } + } + + "work with elseif chains" in { + val cpg = code(""" + condition1.name shouldBe "cond1" + condition1.code shouldBe "$cond1" + condition1.lineNumber shouldBe Some(2) + } + + val elseif1 = inside(ifAst.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + inside(thenBlock.astChildren.l) { case List(body1: Identifier) => + body1.name shouldBe "body1" + body1.code shouldBe "$body1" + body1.lineNumber shouldBe Some(3) + } + + inside(elseBlock.astChildren.l) { case List(elseStructure: ControlStructure) => + elseStructure + } + } + + elseif1.lineNumber shouldBe Some(4) + + inside(elseif1.condition.l) { case List(condition2: Identifier) => + condition2.name shouldBe "cond2" + condition2.code shouldBe "$cond2" + condition2.lineNumber shouldBe Some(4) + } + + val elseif2 = inside(elseif1.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + inside(thenBlock.astChildren.l) { case List(body2: Identifier) => + body2.name shouldBe "body2" + body2.code shouldBe "$body2" + body2.lineNumber shouldBe Some(5) + } + + inside(elseBlock.astChildren.l) { case List(elseStructure: ControlStructure) => + elseStructure + } + } + + elseif2.lineNumber shouldBe Some(6) + + inside(elseif2.condition.l) { case List(condition3: Identifier) => + condition3.name shouldBe "cond3" + condition3.code shouldBe "$cond3" + condition3.lineNumber shouldBe Some(6) + } + + inside(elseif2.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + thenBlock.lineNumber shouldBe Some(6) + elseBlock.lineNumber shouldBe Some(8) + + inside(thenBlock.astChildren.l) { case List(body3: Identifier) => + body3.name shouldBe "body3" + body3.code shouldBe "$body3" + body3.lineNumber shouldBe Some(7) + } + + inside(elseBlock.astChildren.l) { case List(body4: Identifier) => + body4.name shouldBe "body4" + body4.code shouldBe "$body4" + body4.lineNumber shouldBe Some(9) + } + } + } + + "work with else...if chains" in { + val cpg = code(""" + condition1.name shouldBe "cond1" + } + + val elseif1 = inside(ifAst.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + inside(thenBlock.astChildren.l) { case List(body1: Identifier) => + body1.name shouldBe "body1" + } + + inside(elseBlock.astChildren.l) { case List(elseStructure: ControlStructure) => + elseStructure + } + } + + inside(elseif1.condition.l) { case List(condition2: Identifier) => + condition2.name shouldBe "cond2" + } + + val elseif2 = inside(elseif1.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + inside(thenBlock.astChildren.l) { case List(body2: Identifier) => + body2.name shouldBe "body2" + } + + inside(elseBlock.astChildren.l) { case List(elseStructure: ControlStructure) => + elseStructure + } + } + + inside(elseif2.condition.l) { case List(condition3: Identifier) => + condition3.name shouldBe "cond3" + } + + inside(elseif2.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + inside(thenBlock.astChildren.l) { case List(body3: Identifier) => + body3.name shouldBe "body3" + } + + inside(elseBlock.astChildren.l) { case List(body4: Identifier) => + body4.name shouldBe "body4" + } + } + } + + "work with elseif chains with colon syntax" in { + val cpg = code(""" + condition1.name shouldBe "cond1" + } + + val elseif1 = inside(ifAst.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + inside(thenBlock.astChildren.l) { case List(body1: Identifier) => + body1.name shouldBe "body1" + } + + inside(elseBlock.astChildren.l) { case List(elseStructure: ControlStructure) => + elseStructure + } + } + + inside(elseif1.condition.l) { case List(condition2: Identifier) => + condition2.name shouldBe "cond2" + } + + val elseif2 = inside(elseif1.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + inside(thenBlock.astChildren.l) { case List(body2: Identifier) => + body2.name shouldBe "body2" + } + + inside(elseBlock.astChildren.l) { case List(elseStructure: ControlStructure) => + elseStructure + } + } + + inside(elseif2.condition.l) { case List(condition3: Identifier) => + condition3.name shouldBe "cond3" + } + + inside(elseif2.astChildren.l) { case List(_, thenBlock: Block, elseBlock: Block) => + inside(thenBlock.astChildren.l) { case List(body3: Identifier) => + body3.name shouldBe "body3" + } + + inside(elseBlock.astChildren.l) { case List(body4: Identifier) => + body4.name shouldBe "body4" + } + } + } + } + + "break statements" should { + "support the default depth 1 break" in { + val cpg = code(" + breakStmt.controlStructureType shouldBe ControlStructureTypes.BREAK + breakStmt.astChildren.isEmpty shouldBe true + } + } + + "support arbitrary depth breaks" in { + val cpg = code(" + breakStmt.controlStructureType shouldBe ControlStructureTypes.BREAK + + inside(breakStmt.astChildren.l) { case List(num: Literal) => + num.code shouldBe "5" + num.typeFullName shouldBe TypeConstants.Int + } + } + } + } + + "continue statements" should { + "support the default depth 1 continue" in { + val cpg = code(" + continueStmt.controlStructureType shouldBe ControlStructureTypes.CONTINUE + continueStmt.astChildren.isEmpty shouldBe true + continueStmt.lineNumber shouldBe Some(2) + } + } + + "support arbitrary depth continues" in { + val cpg = code(" + continueStmt.controlStructureType shouldBe ControlStructureTypes.CONTINUE + + inside(continueStmt.astChildren.l) { case List(num: Literal) => + num.code shouldBe "5" + num.typeFullName shouldBe TypeConstants.Int + } + } + } + } + + "while statements" should { + "work with an empty body" in { + val cpg = code(" whileAst + } + + whileAst.code shouldBe "while ($a)" + + inside(whileAst.condition.l) { case List(aIdent: Identifier) => + aIdent.name shouldBe "a" + aIdent.order shouldBe 1 + } + + inside(whileAst.astChildren.collectAll[Block].l) { case List(block) => + block.order shouldBe 2 + block.astChildren.size shouldBe 0 + } + } + + "work with a non-empty body" in { + val cpg = code(""" whileAst + } + + whileAst.code shouldBe "while ($a)" + whileAst.lineNumber shouldBe Some(2) + + inside(whileAst.condition.l) { case List(aIdent: Identifier) => + aIdent.name shouldBe "a" + aIdent.code shouldBe "$a" + aIdent.lineNumber shouldBe Some(2) + } + + inside(whileAst.astChildren.collectAll[Block].l) { case List(block) => + block.lineNumber shouldBe Some(2) + + inside(block.astChildren.l) { case List(bIdent: Identifier, cIdent: Identifier) => + bIdent.name shouldBe "b" + bIdent.code shouldBe "$b" + bIdent.lineNumber shouldBe Some(3) + + cIdent.name shouldBe "c" + cIdent.code shouldBe "$c" + cIdent.lineNumber shouldBe Some(4) + } + } + } + } + + "do statements" should { + "work with an empty body" in { + val cpg = code(" doAst + } + + doASt.code shouldBe "do {...} while ($a)" + + inside(doASt.astChildren.collectAll[Block].l) { case List(block) => + block.order shouldBe 1 + block.astChildren.size shouldBe 0 + } + + inside(doASt.condition.l) { case List(aIdent: Identifier) => + aIdent.name shouldBe "a" + aIdent.order shouldBe 2 + } + } + + "work with a non-empty body" in { + val cpg = code(""" doAst + } + + doAst.code shouldBe "do {...} while ($a)" + doAst.lineNumber shouldBe Some(2) + + inside(doAst.astChildren.collectAll[Block].l) { case List(block) => + block.lineNumber shouldBe Some(2) + + inside(block.astChildren.l) { case List(bIdent: Identifier, cIdent: Identifier) => + bIdent.name shouldBe "b" + bIdent.code shouldBe "$b" + bIdent.lineNumber shouldBe Some(3) + + cIdent.name shouldBe "c" + cIdent.code shouldBe "$c" + cIdent.lineNumber shouldBe Some(4) + } + } + + inside(doAst.condition.l) { case List(aIdent: Identifier) => + aIdent.name shouldBe "a" + aIdent.code shouldBe "$a" + aIdent.lineNumber shouldBe Some(5) + } + } + } + + "for statements with the usual format" should { + val cpg = code(""" + iLocal.name shouldBe "i" + iLocal.code shouldBe "$i" + } + } + + "create the FOR control structure" in { + inside(cpg.controlStructure.l) { case List(forStructure) => + forStructure.controlStructureType shouldBe ControlStructureTypes.FOR + forStructure.code shouldBe "for ($i = 0;$i < 42;$i++)" + forStructure.lineNumber shouldBe Some(2) + } + } + + "create the correct initialiser AST" in { + inside(cpg.controlStructure.astChildren.l) { case List(initialiser, _, _, _) => + initialiser.code shouldBe "$i = 0" + initialiser.lineNumber shouldBe Some(2) + } + } + + "create the correct condition AST" in { + inside(cpg.controlStructure.astChildren.l) { case List(_, condition, _, _) => + condition.code shouldBe "$i < 42" + condition.lineNumber shouldBe Some(2) + + cpg.controlStructure.condition.l shouldBe List(condition) + } + } + + "create the correct update AST" in { + inside(cpg.controlStructure.astChildren.l) { case List(_, _, update, _) => + update.code shouldBe "$i++" + update.lineNumber shouldBe Some(2) + } + } + + "create the correct body AST" in { + inside(cpg.controlStructure.astChildren.l) { case List(_, _, _, body: Block) => + body.astChildren.code.l shouldBe List("echo $i") + } + } + } + + "for statements with multiple inits, conditions and updates" should { + val cpg = code(""" 42; $i++, $j--) { + | echo $i; + |} + |""".stripMargin) + + "add a local for the initializer to the enclosing method" in { + inside(cpg.local.sortBy(_.name).toList) { case List(iLocal, jLocal) => + iLocal.name shouldBe "i" + iLocal.code shouldBe "$i" + + jLocal.name shouldBe "j" + jLocal.code shouldBe "$j" + } + } + + "create the FOR control structure" in { + inside(cpg.controlStructure.l) { case List(forStructure) => + forStructure.controlStructureType shouldBe ControlStructureTypes.FOR + forStructure.code shouldBe "for ($i = 0,$j = 100;$i < 42,$j > 42;$i++,$j--)" + forStructure.lineNumber shouldBe Some(2) + } + } + + "create the correct initialiser AST" in { + inside(cpg.controlStructure.astChildren.l) { case List(initialisers: Block, _, _, _) => + inside(initialisers.astChildren.l) { case List(iInit, jInit) => + iInit.code shouldBe "$i = 0" + iInit.lineNumber shouldBe Some(2) + + jInit.code shouldBe "$j = 100" + jInit.lineNumber shouldBe Some(2) + } + } + } + + "create the correct condition AST" in { + inside(cpg.controlStructure.astChildren.l) { case List(_, conditions: Block, _, _) => + inside(conditions.astChildren.l) { case List(iCond, jCond) => + iCond.code shouldBe "$i < 42" + iCond.lineNumber shouldBe Some(2) + + jCond.code shouldBe "$j > 42" + jCond.lineNumber shouldBe Some(2) + } + } + } + + "create the correct update AST" in { + inside(cpg.controlStructure.astChildren.l) { case List(_, _, updates: Block, _) => + inside(updates.astChildren.l) { case List(iUpdate, jUpdate) => + iUpdate.code shouldBe "$i++" + iUpdate.lineNumber shouldBe Some(2) + + jUpdate.code shouldBe "$j--" + jUpdate.lineNumber shouldBe Some(2) + } + } + } + + "create the correct body AST" in { + inside(cpg.controlStructure.astChildren.l) { case List(_, _, _, body: Block) => + body.astChildren.code.l shouldBe List("echo $i") + } + } + } + + "a full try-catch-finally chain" should { + val cpg = code(""" + tryStructure.controlStructureType shouldBe ControlStructureTypes.TRY + tryStructure.lineNumber shouldBe Some(2) + + inside(tryStructure.astChildren.l) { case List(body: Block, _, _, _) => + body.order shouldBe 1 + inside(body.astChildren.code.l) { case List(bodyCode) => + bodyCode shouldBe "$body1" + } + } + } + } + + "create the catch blocks correctly" in { + val catchBlocks = cpg.controlStructure.astChildren.order(2).toSet + + catchBlocks.flatMap(_.astChildren.code.toSet) shouldBe Set("$body2", "$body3") + catchBlocks.flatMap(_.lineNumber) shouldBe Set(4, 6) + } + + "create the finally block correctly" in { + inside(cpg.controlStructure.astChildren.order(3).l) { case List(finallyBlock) => + finallyBlock.astChildren.code.toSet shouldBe Set("$body4") + finallyBlock.lineNumber shouldBe Some(8) + } + } + } + + "a try-finally chain" should { + val cpg = code(""" + tryStructure.controlStructureType shouldBe ControlStructureTypes.TRY + tryStructure.lineNumber shouldBe Some(2) + + inside(tryStructure.astChildren.l) { case List(body: Block, _) => + body.order shouldBe 1 + inside(body.astChildren.code.l) { case List(bodyCode) => + bodyCode shouldBe "$body1" + } + } + } + } + + "create the finally block correctly" in { + inside(cpg.controlStructure.astChildren.order(3).l) { case List(finallyBlock) => + finallyBlock.astChildren.code.toSet shouldBe Set("$body4") + finallyBlock.lineNumber shouldBe Some(4) + } + } + } + + "a try-catch chain without finally" should { + val cpg = code(""" + tryStructure.controlStructureType shouldBe ControlStructureTypes.TRY + tryStructure.lineNumber shouldBe Some(2) + + inside(tryStructure.astChildren.l) { case List(body: Block, _, _) => + body.order shouldBe 1 + inside(body.astChildren.code.l) { case List(bodyCode) => + bodyCode shouldBe "$body1" + } + } + } + } + + "create the catch blocks correctly" in { + val catchBlocks = cpg.controlStructure.astChildren.order(2).toSet + + catchBlocks.flatMap(_.astChildren.code.toSet) shouldBe Set("$body2", "$body3") + catchBlocks.flatMap(_.lineNumber) shouldBe Set(4, 6) + } + } + + "a throw in a try-catch should be created correctly" in { + val cpg = code(""" + throwExpr.lineNumber shouldBe Some(3) + throwExpr.code shouldBe "throw $x" + throwExpr.astChildren.code.l shouldBe List("$x") + } + } + + "goto statements and labels" should { + val cpg = code(""" + goto.controlStructureType shouldBe ControlStructureTypes.GOTO + goto.code shouldBe "goto TARGET" + goto.lineNumber shouldBe Some(2) + + inside(goto.astChildren.l) { case List(jumpLabel: JumpLabel) => + jumpLabel.name shouldBe "TARGET" + jumpLabel.code shouldBe "TARGET" + jumpLabel.lineNumber shouldBe Some(2) + jumpLabel.order shouldBe 1 // Important for CFG creation + } + } + } + + "create the correct jumpTarget" in { + inside(cpg.jumpTarget.l) { case List(jumpTarget) => + jumpTarget.name shouldBe "TARGET" + jumpTarget.code shouldBe "TARGET" + jumpTarget.lineNumber shouldBe Some(3) + } + } + } + + "match expressions" should { + "work without a default case" in { + val cpg = code(""" "A", + | $b, $c => "NOT A", + |} + |""".stripMargin) + + inside(cpg.controlStructure.l) { case List(matchStructure) => + matchStructure.controlStructureType shouldBe ControlStructureTypes.MATCH + matchStructure.code shouldBe "match ($condition)" + matchStructure.lineNumber shouldBe Some(2) + + inside(matchStructure.condition.l) { case List(condition: Identifier) => + condition.name shouldBe "condition" + condition.code shouldBe "$condition" + condition.lineNumber shouldBe Some(2) + } + + inside(matchStructure.astChildren.collectAll[Block].astChildren.l) { + case List( + aTarget: JumpTarget, + aValue: Literal, + bTarget: JumpTarget, + cTarget: JumpTarget, + otherValue: Literal + ) => + aTarget.code shouldBe "case $a" + aTarget.lineNumber shouldBe Some(3) + + aValue.code shouldBe "\"A\"" + aValue.lineNumber shouldBe Some(3) + + bTarget.code shouldBe "case $b" + bTarget.lineNumber shouldBe Some(4) + + cTarget.code shouldBe "case $c" + cTarget.lineNumber shouldBe Some(4) + + otherValue.code shouldBe "\"NOT A\"" + otherValue.lineNumber shouldBe Some(4) + } + } + } + } + + "work with a default case" in { + val cpg = code(""" "A", + | $b, $c => "NOT A", + | default => "DEFAULT", + |} + |""".stripMargin) + + inside(cpg.controlStructure.l) { case List(matchStructure) => + matchStructure.controlStructureType shouldBe ControlStructureTypes.MATCH + matchStructure.code shouldBe "match ($condition)" + matchStructure.lineNumber shouldBe Some(2) + + inside(matchStructure.condition.l) { case List(condition: Identifier) => + condition.name shouldBe "condition" + condition.code shouldBe "$condition" + condition.lineNumber shouldBe Some(2) + } + + inside(matchStructure.astChildren.collectAll[Block].astChildren.l) { + case List( + aTarget: JumpTarget, + aValue: Literal, + bTarget: JumpTarget, + cTarget: JumpTarget, + otherValue: Literal, + defaultTarget: JumpTarget, + defaultValue: Literal + ) => + aTarget.code shouldBe "case $a" + aTarget.lineNumber shouldBe Some(3) + + aValue.code shouldBe "\"A\"" + aValue.lineNumber shouldBe Some(3) + + bTarget.code shouldBe "case $b" + bTarget.lineNumber shouldBe Some(4) + + cTarget.code shouldBe "case $c" + cTarget.lineNumber shouldBe Some(4) + + otherValue.code shouldBe "\"NOT A\"" + otherValue.lineNumber shouldBe Some(4) + + defaultTarget.code shouldBe "default" + defaultTarget.lineNumber shouldBe Some(5) + + defaultValue.code shouldBe "\"DEFAULT\"" + defaultValue.lineNumber shouldBe Some(5) + } + } + } + + "yield from should be represented as a yield with the correct code field" in { + val cpg = code(""" + yieldStructure.controlStructureType shouldBe ControlStructureTypes.YIELD + yieldStructure.code shouldBe "yield from $xs" + yieldStructure.lineNumber shouldBe Some(3) + + inside(yieldStructure.astChildren.l) { case List(xs: Identifier) => + xs.name shouldBe "xs" + xs.code shouldBe "$xs" + xs.lineNumber shouldBe Some(3) + } + } + } + + "yield expressions" should { + "be created when they have no value" in { + val cpg = code(""" + yieldStructure.controlStructureType shouldBe ControlStructureTypes.YIELD + yieldStructure.code shouldBe "yield" + yieldStructure.lineNumber shouldBe Some(3) + + yieldStructure.astChildren.size shouldBe 0 + } + } + + "be created when they have values without keys" in { + val cpg = code(""" + yieldStructure.controlStructureType shouldBe ControlStructureTypes.YIELD + yieldStructure.code shouldBe "yield 1" + yieldStructure.lineNumber shouldBe Some(3) + + inside(yieldStructure.astChildren.l) { case List(value: Literal) => + value.code shouldBe "1" + value.lineNumber shouldBe Some(3) + } + } + } + + "be created when they have values with keys" in { + val cpg = code(""" $x; + |} + |""".stripMargin) + + inside(cpg.controlStructure.l) { case List(yieldStructure) => + yieldStructure.controlStructureType shouldBe ControlStructureTypes.YIELD + yieldStructure.code shouldBe "yield 1 => $x" + yieldStructure.lineNumber shouldBe Some(3) + + inside(yieldStructure.astChildren.l) { case List(key: Literal, value: Identifier) => + key.code shouldBe "1" + key.lineNumber shouldBe Some(3) + + value.name shouldBe "x" + value.code shouldBe "$x" + value.lineNumber shouldBe Some(3) + } + } + } + } + + "foreach statements should not create parentless identifiers" in { + val cpg = code(""" Try(node.astParent).isFailure).toList shouldBe Nil + } + + "foreach statements with only simple values should be represented as a for" in { + val cpg = code(""" + iterLocal.name shouldBe "iter_tmp0" + valLocal.name shouldBe "val" + + foreachStruct + } + + foreachStruct.code shouldBe "foreach ($arr as $val)" + + val (initAsts, conditionAst, updateAsts, body) = inside(foreachStruct.astChildren.l) { + case List(initAsts: Block, conditionAst: Call, updateAsts: Block, body: Block) => + (initAsts, conditionAst, updateAsts, body) + } + + inside(initAsts.astChildren.l) { case List(iterInit: Call, valInit: Call) => + iterInit.name shouldBe Operators.assignment + iterInit.code shouldBe "$iter_tmp0 = $arr" + inside(iterInit.argument.l) { case List(iterTemp: Identifier, iterExpr: Identifier) => + iterTemp.name shouldBe "iter_tmp0" + iterTemp.code shouldBe "$iter_tmp0" + iterTemp.argumentIndex shouldBe 1 + + iterExpr.name shouldBe "arr" + iterExpr.code shouldBe "$arr" + iterExpr.argumentIndex shouldBe 2 + } + + valInit.name shouldBe Operators.assignment + valInit.code shouldBe "$val = $iter_tmp0->current()" + inside(valInit.argument.l) { case List(valId: Identifier, currentCall: Call) => + valId.name shouldBe "val" + valId.code shouldBe "$val" + valId.argumentIndex shouldBe 1 + + currentCall.name shouldBe "current" + currentCall.methodFullName shouldBe s"Iterator.current" + currentCall.code shouldBe "$iter_tmp0->current()" + inside(currentCall.argument(0).start.l) { case List(iterRecv: Identifier) => + iterRecv.name shouldBe "iter_tmp0" + iterRecv.argumentIndex shouldBe 0 + } + } + } + + conditionAst.name shouldBe Operators.logicalNot + conditionAst.code shouldBe "!is_null($val)" + inside(conditionAst.astChildren.l) { case List(isNullCall: Call) => + isNullCall.name shouldBe "is_null" + isNullCall.code shouldBe "is_null($val)" + } + + inside(updateAsts.astChildren.l) { case List(nextCall: Call, valAssign: Call) => + nextCall.name shouldBe "next" + nextCall.methodFullName shouldBe "Iterator.next" + nextCall.code shouldBe "$iter_tmp0->next()" + inside(nextCall.argument(0).start.l) { case List(iterTmp: Identifier) => + iterTmp.name shouldBe "iter_tmp0" + iterTmp.code shouldBe "$iter_tmp0" + iterTmp.argumentIndex shouldBe 0 + } + + valAssign.name shouldBe Operators.assignment + valAssign.code shouldBe "$val = $iter_tmp0->current()" + } + + inside(body.astChildren.l) { case List(echoCall: Call) => + echoCall.code shouldBe "echo $val" + } + } + + "foreach statements with assignments by ref should be represented as a for" in { + val cpg = code(""" + iterLocal.name shouldBe "iter_tmp0" + valLocal.name shouldBe "val" + + foreachStruct + } + + foreachStruct.code shouldBe "foreach ($arr as &$val)" + + val (initAsts, updateAsts, body) = inside(foreachStruct.astChildren.l) { + case List(initAsts: Block, _, updateAsts: Block, body: Block) => + (initAsts, updateAsts, body) + } + + inside(initAsts.astChildren.l) { case List(_: Call, valInit: Call) => + valInit.name shouldBe Operators.assignment + valInit.code shouldBe "$val = &$iter_tmp0->current()" + inside(valInit.argument.l) { case List(valId: Identifier, addressOfCall: Call) => + valId.name shouldBe "val" + valId.code shouldBe "$val" + valId.argumentIndex shouldBe 1 + + addressOfCall.name shouldBe Operators.addressOf + addressOfCall.code shouldBe "&$iter_tmp0->current()" + + inside(addressOfCall.argument.l) { case List(currentCall: Call) => + currentCall.name shouldBe "current" + currentCall.methodFullName shouldBe s"Iterator.current" + currentCall.code shouldBe "$iter_tmp0->current()" + inside(currentCall.argument(0).start.l) { case List(iterRecv: Identifier) => + iterRecv.name shouldBe "iter_tmp0" + iterRecv.argumentIndex shouldBe 0 + } + } + } + } + + inside(updateAsts.astChildren.l) { case List(_: Call, valAssign: Call) => + valAssign.name shouldBe Operators.assignment + valAssign.code shouldBe "$val = &$iter_tmp0->current()" + } + + inside(body.astChildren.l) { case List(echoCall: Call) => + echoCall.code shouldBe "echo $val" + } + } + + "foreach statements with key-val should be represented as a for" in { + val cpg = code(""" $val) { + | echo $val; + | } + |} + |""".stripMargin) + + val foreachStruct = inside(cpg.method.name("foo").body.astChildren.l) { + case List(iterLocal: Local, keyLocal: Local, valLocal: Local, foreachStruct: ControlStructure) => + iterLocal.name shouldBe "iter_tmp0" + keyLocal.name shouldBe "key" + valLocal.name shouldBe "val" + + foreachStruct + } + + foreachStruct.code shouldBe "foreach ($arr as $key => $val)" + + val (initAsts, updateAsts, body) = inside(foreachStruct.astChildren.l) { + case List(initAsts: Block, _, updateAsts: Block, body: Block) => + (initAsts, updateAsts, body) + } + + inside(initAsts.astChildren.l) { case List(_: Call, valInit: Call) => + valInit.name shouldBe Operators.assignment + valInit.code shouldBe "$key => $val = $iter_tmp0->current()" + inside(valInit.argument.l) { case List(valPair: Call, currentCall: Call) => + valPair.name shouldBe PhpOperators.doubleArrow + valPair.code shouldBe "$key => $val" + inside(valPair.argument.l) { case List(keyId: Identifier, valId: Identifier) => + keyId.name shouldBe "key" + valId.name shouldBe "val" + } + + currentCall.name shouldBe "current" + currentCall.methodFullName shouldBe s"Iterator.current" + currentCall.code shouldBe "$iter_tmp0->current()" + inside(currentCall.argument(0).start.l) { case List(iterRecv: Identifier) => + iterRecv.name shouldBe "iter_tmp0" + iterRecv.argumentIndex shouldBe 0 + } + } + } + + inside(updateAsts.astChildren.l) { case List(_: Call, valAssign: Call) => + valAssign.name shouldBe Operators.assignment + valAssign.code shouldBe "$key => $val = $iter_tmp0->current()" + } + + inside(body.astChildren.l) { case List(echoCall: Call) => + echoCall.code shouldBe "echo $val" + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/FieldAccessTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/FieldAccessTests.scala new file mode 100644 index 00000000..85a8cd20 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/FieldAccessTests.scala @@ -0,0 +1,94 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.nodes.{FieldIdentifier, Identifier} +import io.shiftleft.semanticcpg.language._ + +class FieldAccessTests extends PhpCode2CpgFixture { + + "simple property fetches should be represented as normal field accesses" in { + val cpg = code("field") + + inside(cpg.call.l) { case List(fieldAccess) => + fieldAccess.name shouldBe Operators.fieldAccess + fieldAccess.methodFullName shouldBe Operators.fieldAccess + fieldAccess.code shouldBe "$obj->field" + fieldAccess.lineNumber shouldBe Some(2) + + inside(fieldAccess.argument.l) { case List(objIdentifier: Identifier, fieldIdentifier: FieldIdentifier) => + objIdentifier.name shouldBe "obj" + objIdentifier.code shouldBe "$obj" + objIdentifier.lineNumber shouldBe Some(2) + + fieldIdentifier.canonicalName shouldBe "field" + fieldIdentifier.code shouldBe "field" + fieldIdentifier.lineNumber shouldBe Some(2) + } + } + } + + "property fetches with expr args should be represented as an arbitrary field access" in { + val cpg = code("$field") + + inside(cpg.call.l) { case List(fieldAccess) => + fieldAccess.name shouldBe Operators.fieldAccess + fieldAccess.methodFullName shouldBe Operators.fieldAccess + fieldAccess.code shouldBe "$$obj->$field" + fieldAccess.lineNumber shouldBe Some(2) + + inside(fieldAccess.argument.l) { case List(objIdentifier: Identifier, field: Identifier) => + objIdentifier.name shouldBe "obj" + objIdentifier.code shouldBe "$$obj" + objIdentifier.lineNumber shouldBe Some(2) + + field.name shouldBe "field" + field.code shouldBe "$field" + field.lineNumber shouldBe Some(2) + } + } + } + + "nullsafe property fetches should be represented as normal field accesses with the correct code" in { + val cpg = code("field") + + inside(cpg.call.l) { case List(fieldAccess) => + fieldAccess.name shouldBe Operators.fieldAccess + fieldAccess.methodFullName shouldBe Operators.fieldAccess + fieldAccess.code shouldBe "$obj?->field" + fieldAccess.lineNumber shouldBe Some(2) + + inside(fieldAccess.argument.l) { case List(objIdentifier: Identifier, fieldIdentifier: FieldIdentifier) => + objIdentifier.name shouldBe "obj" + objIdentifier.code shouldBe "$obj" + objIdentifier.lineNumber shouldBe Some(2) + + fieldIdentifier.canonicalName shouldBe "field" + fieldIdentifier.code shouldBe "field" + fieldIdentifier.lineNumber shouldBe Some(2) + } + } + } + + "static property fetches should be represented as normal field accesses with the correct code" in { + val cpg = code(" + fieldAccess.name shouldBe Operators.fieldAccess + fieldAccess.methodFullName shouldBe Operators.fieldAccess + fieldAccess.code shouldBe "className::$field" + fieldAccess.lineNumber shouldBe Some(2) + + inside(fieldAccess.argument.l) { case List(classIdentifier: Identifier, fieldIdentifier: Identifier) => + classIdentifier.name shouldBe "className" + classIdentifier.code shouldBe "className" + classIdentifier.lineNumber shouldBe Some(2) + + fieldIdentifier.name shouldBe "field" + fieldIdentifier.code shouldBe "$field" + fieldIdentifier.lineNumber shouldBe Some(2) + } + } + } + +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/LocalTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/LocalTests.scala new file mode 100644 index 00000000..8c68e4cf --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/LocalTests.scala @@ -0,0 +1,81 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.shiftleft.semanticcpg.language._ + +class LocalTests extends PhpCode2CpgFixture { + + "locals for methods" should { + "be created for methods with assigns" in { + val cpg = code(""" + xLocal.name shouldBe "x" + xLocal.code shouldBe "$x" + + yLocal.name shouldBe "y" + yLocal.code shouldBe "$y" + } + } + + "have ref edges from uses" in { + val cpg = code(""" + xIdent1._localViaRefOut.map(_.name) should contain("x") + xIdent2._localViaRefOut.map(_.name) should contain("x") + } + } + + "not be created if the variable matches a parameter type" in { + val cpg = code(""" + xLocal.name shouldBe "x" + xLocal.code shouldBe "$x" + } + } + } + + "nested static locals should be created correctly" in { + val cpg = code(""" + xLocal.name shouldBe "x" + xLocal.code shouldBe "static $x" + xLocal.lineNumber shouldBe Some(4) + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/MemberTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/MemberTests.scala new file mode 100644 index 00000000..67a16e33 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/MemberTests.scala @@ -0,0 +1,223 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.parser.Domain +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.appthreat.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.{ModifierTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, Literal} +import io.shiftleft.semanticcpg.language._ + +class MemberTests extends PhpCode2CpgFixture { + + "class constants" should { + val cpg = code(""" + aMember.name shouldBe "A" + aMember.code shouldBe "const A" + + bMember.name shouldBe "B" + bMember.code shouldBe "const B" + + cMember.name shouldBe "C" + cMember.code shouldBe "const C" + } + } + + "have an access modifier node for the C constant" in { + inside(cpg.member("C").modifier.sortBy(_.modifierType).toList) { case List(finalModifier, publicModifier) => + finalModifier.modifierType shouldBe ModifierTypes.FINAL + publicModifier.modifierType shouldBe ModifierTypes.PUBLIC + } + } + + "have a clinit method with the constant initializers" in { + + inside(cpg.method.nameExact(Defines.StaticInitMethodName).l) { case List(clinitMethod) => + inside(clinitMethod.body.astChildren.l) { case List(aAssign: Call, bAssign: Call, cAssign: Call) => + checkConstAssign(aAssign, "A") + checkConstAssign(bAssign, "B") + checkConstAssign(cAssign, "C") + } + } + } + } + + "class properties (fields)" should { + val cpg = code(""" + aField.name shouldBe "a" + aField.code shouldBe "$a" + aField.modifier.modifierType.l shouldBe List(ModifierTypes.PUBLIC) + + bField.name shouldBe "b" + bField.code shouldBe "$b" + bField.modifier.modifierType.l shouldBe List(ModifierTypes.PUBLIC) + + cField.name shouldBe "c" + cField.code shouldBe "$c" + inside(cField.modifier.modifierType.l.sorted) { case List(finalModifier, protectedModifier) => + finalModifier shouldBe ModifierTypes.FINAL + protectedModifier shouldBe ModifierTypes.PROTECTED + } + } + } + + "have assignments added to the default constructor" in { + inside(cpg.method.nameExact(Domain.ConstructorMethodName).l) { case List(initMethod) => + inside(initMethod.body.astChildren.l) { case List(aAssign: Call, bAssign: Call, cAssign: Call) => + checkFieldAssign(aAssign, "a") + checkFieldAssign(bAssign, "b") + checkFieldAssign(cAssign, "c") + } + } + } + } + + "class properties (fields) for classes with a constructor" should { + + val cpg = code(""" + aField.name shouldBe "a" + aField.code shouldBe "$a" + aField.modifier.modifierType.l shouldBe List(ModifierTypes.PUBLIC) + + bField.name shouldBe "b" + bField.code shouldBe "$b" + bField.modifier.modifierType.l shouldBe List(ModifierTypes.PUBLIC) + + cField.name shouldBe "c" + cField.code shouldBe "$c" + inside(cField.modifier.modifierType.l.sorted) { case List(finalModifier, protectedModifier) => + finalModifier shouldBe ModifierTypes.FINAL + protectedModifier shouldBe ModifierTypes.PROTECTED + } + } + } + + "have assignments added to the default constructor" in { + inside(cpg.method.nameExact(Domain.ConstructorMethodName).l) { case List(initMethod) => + inside(initMethod.body.astChildren.l) { case List(aAssign: Call, bAssign: Call, cAssign: Call) => + checkFieldAssign(aAssign, "a") + checkFieldAssign(bAssign, "b") + checkFieldAssign(cAssign, "c") + } + } + } + } + + "class const accesses should be created with the correct field access" in { + val cpg = code(" + fieldAccess.code shouldBe "Foo::X" + fieldAccess.lineNumber shouldBe Some(2) + + inside(fieldAccess.argument.l) { case List(fooArg: Identifier, xArg: FieldIdentifier) => + fooArg.name shouldBe "Foo" + fooArg.code shouldBe "Foo" + fooArg.argumentIndex shouldBe 1 + fooArg.lineNumber shouldBe Some(2) + + xArg.canonicalName shouldBe "X" + xArg.code shouldBe "X" + xArg.argumentIndex shouldBe 2 + xArg.lineNumber shouldBe Some(2) + } + } + } + + "non-class consts" should { + val cpg = code(""" + xMember.name shouldBe "X" + xMember.lineNumber shouldBe Some(2) + xMember.typeDecl.name shouldBe "" + + inside(xMember.modifier.l) { case List(finalModifier) => + finalModifier.modifierType shouldBe ModifierTypes.FINAL + } + } + } + + "be initialized in the global typedecl's method" in { + inside(cpg.call.nameExact(Operators.assignment).l) { case List(assign) => + checkConstAssign(assign, "X") + } + } + + "be fetched as a field access for a global field" in { + inside(cpg.call.name("echo").argument.l) { case List(constFetch: Call) => + constFetch.name shouldBe Operators.fieldAccess + constFetch.lineNumber shouldBe Some(3) + constFetch.code shouldBe "X" + + inside(constFetch.argument.l) { case List(globalIdentifier: Identifier, fieldIdentifier: FieldIdentifier) => + globalIdentifier.name shouldBe "" + fieldIdentifier.canonicalName shouldBe "X" + } + } + } + } + + private def checkFieldAssign(assign: Call, expectedValue: String): Unit = { + assign.name shouldBe Operators.assignment + assign.methodFullName shouldBe Operators.assignment + + inside(assign.argument.l) { case List(targetFa: Call, source: Literal) => + targetFa.name shouldBe Operators.fieldAccess + inside(targetFa.argument.l) { case List(identifier: Identifier, fieldIdentifier: FieldIdentifier) => + identifier.name shouldBe "this" + identifier.code shouldBe "$this" + identifier.argumentIndex shouldBe 1 + + fieldIdentifier.canonicalName shouldBe expectedValue + fieldIdentifier.code shouldBe expectedValue + fieldIdentifier.argumentIndex shouldBe 2 + } + + source.code shouldBe s"\"$expectedValue\"" + source.argumentIndex shouldBe 2 + } + } + + private def checkConstAssign(assign: Call, expectedValue: String): Unit = { + assign.name shouldBe Operators.assignment + assign.methodFullName shouldBe Operators.assignment + + inside(assign.argument.l) { case List(target: Identifier, source: Literal) => + target.name shouldBe expectedValue + target.code shouldBe expectedValue + target.argumentIndex shouldBe 1 + + source.code shouldBe s"\"$expectedValue\"" + source.argumentIndex shouldBe 2 + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/MethodTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/MethodTests.scala new file mode 100644 index 00000000..93aced04 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/MethodTests.scala @@ -0,0 +1,155 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.appthreat.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.{ModifierTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal, Local} +import io.shiftleft.semanticcpg.language._ + +class MethodTests extends PhpCode2CpgFixture { + + "method nodes should be created with the correct fields" in { + val cpg = code( + """ + fooMethod.fullName shouldBe s"foo" + fooMethod.signature shouldBe s"${Defines.UnresolvedSignature}(0)" + fooMethod.lineNumber shouldBe Some(2) + fooMethod.code shouldBe "function foo()" + fooMethod.astParentType shouldBe "METHOD" + fooMethod.astParentFullName.endsWith("") shouldBe true + + inside(fooMethod.methodReturn.start.l) { case List(methodReturn) => + methodReturn.typeFullName shouldBe "int" + methodReturn.code shouldBe "RET" + methodReturn.lineNumber shouldBe Some(2) + } + } + } + + "static variables without default values should be represented as the correct local nodes" in { + val cpg = code(""" + xLocal.name shouldBe "x" + xLocal.code shouldBe "static $x" + xLocal.lineNumber shouldBe Some(3) + + yLocal.name shouldBe "y" + yLocal.code shouldBe "static $y" + yLocal.lineNumber shouldBe Some(3) + } + } + + "static variables with default values should have the correct initialisers" in { + val cpg = code(""" + xLocal.name shouldBe "x" + xLocal.code shouldBe "static $x" + xLocal.lineNumber shouldBe Some(3) + + yLocal.name shouldBe "y" + yLocal.code shouldBe "static $y" + yLocal.lineNumber shouldBe Some(3) + + xAssign.name shouldBe Operators.assignment + xAssign.code shouldBe "static $x = 42" + inside(xAssign.argument.l) { case List(xIdent: Identifier, literal: Literal) => + xIdent.name shouldBe "x" + xIdent.code shouldBe "$x" + xIdent.lineNumber shouldBe Some(3) + + literal.code shouldBe "42" + literal.lineNumber shouldBe Some(3) + } + } + } + + "methods should be accessible from the file node" in { + val cpg = code( + """", "foo") + cpg.method.name("foo").filename.l shouldBe List("test.php") + } + + "global method full name should include the file for uniqueness" in { + val cpg = code("").fullName.l shouldBe List("test.php:") + } + + "explicit constructors" should { + val cpg = code( + """ + constructor.modifier.modifierType.toSet shouldBe Set(ModifierTypes.CONSTRUCTOR, ModifierTypes.PUBLIC) + } + + cpg.method.name("__construct").size shouldBe cpg.method.isConstructor.size + } + + "have a filename set with traversal to the file" in { + inside(cpg.method.nameExact("__construct").l) { case List(constructor) => + constructor.filename shouldBe "foo.php" + constructor.file.name.l shouldBe List("foo.php") + } + } + } + + "default constructors" should { + val cpg = code( + """ + constructor.modifier.modifierType.toSet shouldBe Set( + ModifierTypes.CONSTRUCTOR, + ModifierTypes.PUBLIC, + ModifierTypes.VIRTUAL + ) + } + + cpg.method.name("__construct").size shouldBe cpg.method.isConstructor.size + } + + "have a filename set with traversal to the file" in { + inside(cpg.method.nameExact("__construct").l) { case List(constructor) => + constructor.filename shouldBe "foo.php" + constructor.file.name.l shouldBe List("foo.php") + } + } + } + +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/NamespaceTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/NamespaceTests.scala new file mode 100644 index 00000000..a55d7c9a --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/NamespaceTests.scala @@ -0,0 +1,163 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.Method + +class NamespaceTests extends PhpCode2CpgFixture { + "namespaces should be able to contain statements as top-level AST children" in { + val cpg = code(""" + ns.astChildren.code.l shouldBe List("echo 0") + } + } + + "methods defined in non-namespaced code should not include a namespace prefix" in { + val cpg = code("""foo") + cpg.method.name("bar").fullName.l shouldBe List("A::bar") + } + + "static and instance methods in namespaced code should be correct" in { + val cpg = code("""foo") + cpg.method.name("bar").fullName.l shouldBe List("ns\\A::bar") + } + + "global namespace block should have the relative filename prepended to fullName" in { + val cpg = code("").fullName.sorted.l shouldBe List( + // The namespace added by the MetaDataPass + "", + // The per-file namespaces actually used + "bar.php:", + "foo.php:" + ) + } + + "global variables should have AST in edges from the enclosing global method" in { + val cpg = code(""" + aLocal.name shouldBe "a" + aLocal.code shouldBe "$a" + aLocal.lineNumber shouldBe Some(1) + + inside(aLocal.method.l) { case List(globalMethod) => + globalMethod.name shouldBe "" + globalMethod.fullName shouldBe "foo.php:" + } + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/OperatorTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/OperatorTests.scala new file mode 100644 index 00000000..2cb5cdbf --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/OperatorTests.scala @@ -0,0 +1,670 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.astcreation.AstCreator.{NameConstants, TypeConstants} +import io.appthreat.php2atom.parser.Domain.{PhpDomainTypeConstants, PhpOperators} +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.appthreat.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal, Local, TypeRef} +import io.shiftleft.passes.IntervalKeyPool +import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal + +class OperatorTests extends PhpCode2CpgFixture { + + val filenameKeyPool = new IntervalKeyPool(first = 0, last = Long.MaxValue) + + "assignment operators" should { + "have the correct arguments set" in { + val cpg = code(""" call + } + + inside(assignment.argument.l) { case List(target: Identifier, source: Literal) => + target.name shouldBe "a" + target.code shouldBe "$a" + target.argumentIndex shouldBe 1 + + source.code shouldBe "2" + source.argumentIndex shouldBe 2 + } + } + + "have the correct method names set" in { + val testData = List( + ("$a = $b", Operators.assignment), + ("$a = &$b", Operators.assignment), + ("$a &= $b", Operators.assignmentAnd), + ("$a |= $b", Operators.assignmentOr), + ("$a ^= $b", Operators.assignmentXor), + ("$a ??= $b", PhpOperators.assignmentCoalesceOp), + ("$a .= $b", PhpOperators.assignmentConcatOp), + ("$a /= $b", Operators.assignmentDivision), + ("$a -= $b", Operators.assignmentMinus), + ("$a %= $b", Operators.assignmentModulo), + ("$a *= $b", Operators.assignmentMultiplication), + ("$a += $b", Operators.assignmentPlus), + ("$a **= $b", Operators.assignmentExponentiation), + ("$a <<= $b", Operators.assignmentShiftLeft), + ("$a >>= $b", Operators.assignmentArithmeticShiftRight) + ) + + testData.foreach { case (testCode, expectedType) => + val cpg = code(s" call + } + + inside(addition.argument.l) { case List(expr: Identifier) => + expr.name shouldBe "a" + expr.code shouldBe "$a" + expr.argumentIndex shouldBe 1 + } + } + + "have the correct method names set" in { + val testData = List( + ("~$a", Operators.not), + ("!$a", Operators.logicalNot), + ("$a--", Operators.postDecrement), + ("$a++", Operators.postIncrement), + ("--$a", Operators.preDecrement), + ("++$a", Operators.preIncrement), + ("-$a", Operators.minus), + ("+$a", Operators.plus) + ) + + testData.foreach { case (testCode, expectedType) => + val cpg = code(s" call + } + + inside(addition.argument.l) { case List(target: Identifier, source: Literal) => + target.name shouldBe "a" + target.code shouldBe "$a" + target.argumentIndex shouldBe 1 + + source.code shouldBe "2" + source.argumentIndex shouldBe 2 + } + } + + "have the correct method names set" in { + val testData = List( + ("1 & 2", Operators.and), + ("1 | 2", Operators.or), + ("1 ^ 2", Operators.xor), + ("$a && $b", Operators.logicalAnd), + ("$a || $b", Operators.logicalOr), + ("$a ?? $b", PhpOperators.coalesceOp), + ("$a . $b", PhpOperators.concatOp), + ("$a / $b", Operators.division), + ("$a == $b", Operators.equals), + ("$a >= $b", Operators.greaterEqualsThan), + ("$a > $b", Operators.greaterThan), + ("$a === $b", PhpOperators.identicalOp), + ("$a and $b", Operators.logicalAnd), + ("$a or $b", Operators.logicalOr), + ("$a xor $b", PhpOperators.logicalXorOp), + ("$a - $b", Operators.minus), + ("$a % $b", Operators.modulo), + ("$a * $b", Operators.multiplication), + ("$a != $b", Operators.notEquals), + ("$a <> $b", Operators.notEquals), + ("$a !== $b", PhpOperators.notIdenticalOp), + ("$a + $b", Operators.plus), + ("$a ** $b", Operators.exponentiation), + ("$a << $b", Operators.shiftLeft), + ("$a >> $b", Operators.arithmeticShiftRight), + ("$a <= $b", Operators.lessEqualsThan), + ("$a < $b", Operators.lessThan), + ("$a <=> $b", PhpOperators.spaceshipOp) + ) + + def normalizeLogicalOps(input: String): String = { + input + .replaceAll(" or ", " || ") + .replaceAll(" and ", " && ") + .replaceAll(" <> ", " != ") + } + + testData.foreach { case (testCode, expectedType) => + val cpg = code(s" + cast + } + + cast.typeFullName shouldBe "int" + cast.code shouldBe "(int) $a" + cast.lineNumber shouldBe Some(2) + + inside(cast.argument.l) { case List(typeRef: TypeRef, expr: Identifier) => + typeRef.typeFullName shouldBe "int" + typeRef.argumentIndex shouldBe 1 + + expr.name shouldBe "a" + expr.argumentIndex shouldBe 2 + } + } + + "have the correct types" in { + val testData = List( + ("(array) $x", PhpDomainTypeConstants.array), + ("(bool) $x", PhpDomainTypeConstants.bool), + ("(double) $x", PhpDomainTypeConstants.double), + ("(int) $x", PhpDomainTypeConstants.int), + ("(object) $x", PhpDomainTypeConstants.obj), + ("(string) $x", PhpDomainTypeConstants.string), + ("(unset) $x", PhpDomainTypeConstants.unset) + ) + + testData.foreach { case (testCode, expectedType) => + val cpg = code(s" + typeRef.typeFullName shouldBe expectedType + typeRef.code shouldBe expectedType + } + } + } + } + + "isset calls" should { + "handle a single argument" in { + val cpg = code(" + call + } + + call.code shouldBe "isset($a)" + call.methodFullName shouldBe PhpOperators.issetFunc + call.typeFullName shouldBe TypeConstants.Bool + call.lineNumber shouldBe Some(2) + + inside(call.argument.l) { case List(arg: Identifier) => + arg.name shouldBe "a" + arg.code shouldBe "$a" + arg.argumentIndex shouldBe 1 + } + } + + "handle multiple arguments" in { + val cpg = code(" + call + } + + // Code is not exactly like the original because we lose the spacing information during parsing + call.code shouldBe "isset($a,$b,$c)" + call.methodFullName shouldBe PhpOperators.issetFunc + call.typeFullName shouldBe TypeConstants.Bool + call.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + call.lineNumber shouldBe Some(2) + + inside(call.argument.l) { case List(aArg: Identifier, bArg: Identifier, cArg: Identifier) => + aArg.name shouldBe "a" + aArg.code shouldBe "$a" + aArg.argumentIndex shouldBe 1 + + bArg.name shouldBe "b" + bArg.code shouldBe "$b" + bArg.argumentIndex shouldBe 2 + + cArg.name shouldBe "c" + cArg.code shouldBe "$c" + cArg.argumentIndex shouldBe 3 + } + } + } + + "print calls should be created correctly" in { + val cpg = code(" + printCall.methodFullName shouldBe "print" + printCall.typeFullName shouldBe TypeConstants.Int + printCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + printCall.lineNumber shouldBe Some(2) + + inside(printCall.argument.l) { case List(arg: Literal) => + arg.code shouldBe "\"Hello, world\"" + } + } + } + + "ternary operators" should { + "be created correctly for general cond ? then : else style operators" in { + val cpg = code(" + conditionalOp + } + + call.methodFullName shouldBe Operators.conditional + call.code shouldBe "$a ? $b : $c" + call.lineNumber shouldBe Some(2) + + inside(call.argument.l) { case List(aArg: Identifier, bArg: Identifier, cArg: Identifier) => + aArg.name shouldBe "a" + aArg.code shouldBe "$a" + aArg.argumentIndex shouldBe 1 + + bArg.name shouldBe "b" + bArg.code shouldBe "$b" + bArg.argumentIndex shouldBe 2 + + cArg.name shouldBe "c" + cArg.code shouldBe "$c" + cArg.argumentIndex shouldBe 3 + } + } + + "be created correctly for the shorthand elvis operator" in { + val cpg = code(" + elvisOp + } + + call.methodFullName shouldBe PhpOperators.elvisOp + call.code shouldBe "$a ?: $b" + call.lineNumber shouldBe Some(2) + + inside(call.argument.l) { case List(aArg: Identifier, bArg: Identifier) => + aArg.name shouldBe "a" + aArg.code shouldBe "$a" + aArg.argumentIndex shouldBe 1 + + bArg.name shouldBe "b" + bArg.code shouldBe "$b" + bArg.argumentIndex shouldBe 2 + } + } + } + + "the clone operator should be represented with the correct call node" in { + val cpg = code(" + cloneCall.name shouldBe "clone" + cloneCall.methodFullName shouldBe PhpOperators.cloneFunc + cloneCall.code shouldBe "clone $x" + cloneCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + cloneCall.lineNumber shouldBe Some(2) + + inside(cloneCall.argument.l) { case List(xArg: Identifier) => + xArg.name shouldBe "x" + xArg.code shouldBe "$x" + } + } + } + + "the empty call should be represented with the correct call node" in { + val cpg = code(" + emptyCall.name shouldBe "empty" + emptyCall.methodFullName shouldBe PhpOperators.emptyFunc + emptyCall.code shouldBe "empty($x)" + emptyCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + emptyCall.lineNumber shouldBe Some(2) + + inside(emptyCall.argument.l) { case List(xArg: Identifier) => + xArg.name shouldBe "x" + xArg.code shouldBe "$x" + } + } + } + + "the eval call should be represented with the correct call node" in { + val cpg = code(" + evalCall.name shouldBe "eval" + evalCall.methodFullName shouldBe PhpOperators.evalFunc + evalCall.code shouldBe "eval($x)" + evalCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + evalCall.lineNumber shouldBe Some(2) + + inside(evalCall.argument.l) { case List(xArg: Identifier) => + xArg.name shouldBe "x" + xArg.code shouldBe "$x" + } + } + } + + "exit statements" should { + "be represented with an empty arg list if no args are given" in { + val cpg = code(" + exitCall.name shouldBe "exit" + exitCall.methodFullName shouldBe PhpOperators.exitFunc + exitCall.typeFullName shouldBe TypeConstants.Void + exitCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + exitCall.lineNumber shouldBe Some(2) + exitCall.argument.size shouldBe 0 + exitCall.astChildren.size shouldBe 0 + } + } + + "be represented with an empty arg list if an empty args list is given" in { + val cpg = code(" + exitCall.name shouldBe "exit" + exitCall.methodFullName shouldBe PhpOperators.exitFunc + exitCall.typeFullName shouldBe TypeConstants.Void + exitCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + exitCall.lineNumber shouldBe Some(2) + exitCall.argument.size shouldBe 0 + exitCall.astChildren.size shouldBe 0 + } + } + + "have the correct arg child if an arg is given" in { + val cpg = code(" + exitCall.name shouldBe "exit" + exitCall.methodFullName shouldBe PhpOperators.exitFunc + exitCall.typeFullName shouldBe TypeConstants.Void + exitCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + exitCall.lineNumber shouldBe Some(2) + + inside(cpg.argument.l) { case List(literal: Literal) => + literal.code shouldBe "0" + } + } + } + } + + "the error suppress operator should work" in { + val cpg = code(" + errorSuppress.methodFullName shouldBe PhpOperators.errorSuppress + errorSuppress.code shouldBe "@foo()" + + inside(errorSuppress.argument.l) { case List(fooCall: Call) => + fooCall.name shouldBe "foo" + fooCall.code shouldBe "foo()" + } + } + } + + "instanceof with a simple class name should work" in { + val cpg = code(" + instanceOfCall.name shouldBe Operators.instanceOf + instanceOfCall.methodFullName shouldBe Operators.instanceOf + instanceOfCall.code shouldBe "$foo instanceof Foo" + + inside(instanceOfCall.argument.l) { case List(obj: Identifier, className: Identifier) => + obj.name shouldBe "foo" + obj.code shouldBe "$foo" + + className.name shouldBe "Foo" + className.code shouldBe "Foo" + } + } + } + + "temporary list implementation should work" in { + // TODO This is a simple placeholder implementation that represents most of the useful information + // in the AST, while being pretty much unusable for dataflow. A better implementation needs to follow. + val cpg = code(" + listCall.methodFullName shouldBe PhpOperators.listFunc + listCall.code shouldBe "list($a,$b)" + listCall.lineNumber shouldBe Some(2) + inside(listCall.argument.l) { case List(aArg: Identifier, bArg: Identifier) => + aArg.name shouldBe "a" + aArg.code shouldBe "$a" + aArg.lineNumber shouldBe Some(2) + + bArg.name shouldBe "b" + bArg.code shouldBe "$b" + bArg.lineNumber shouldBe Some(2) + } + } + } + + "include calls" should { + "be correctly represented for normal includes" in { + val cpg = code(" + includeCall.name shouldBe "include" + includeCall.methodFullName shouldBe "include" + includeCall.code shouldBe "include \"path\"" + inside(includeCall.argument.l) { case List(pathLiteral: Literal) => + pathLiteral.code shouldBe "\"path\"" + } + } + } + + "be correctly represented for include_once" in { + val cpg = code(" + includeOnceCall.name shouldBe "include_once" + includeOnceCall.methodFullName shouldBe "include_once" + includeOnceCall.code shouldBe "include_once \"path\"" + inside(includeOnceCall.argument.l) { case List(pathLiteral: Literal) => + pathLiteral.code shouldBe "\"path\"" + } + } + } + + "be correctly represented for normal requires" in { + val cpg = code(" + requireCall.name shouldBe "require" + requireCall.methodFullName shouldBe "require" + requireCall.code shouldBe "require \"path\"" + inside(requireCall.argument.l) { case List(pathLiteral: Literal) => + pathLiteral.code shouldBe "\"path\"" + } + } + } + + "be correctly represented for require once" in { + val cpg = code(" + requireOnce.name shouldBe "require_once" + requireOnce.methodFullName shouldBe "require_once" + requireOnce.code shouldBe "require_once \"path\"" + inside(requireOnce.argument.l) { case List(pathLiteral: Literal) => + pathLiteral.code shouldBe "\"path\"" + } + } + } + } + + "declare calls without statements should be correctly represented" in { + val cpg = code(" + inside(globalMethod.body.astChildren.l) { case List(declareCall: Call) => + declareCall + } + } + + declareCall.name shouldBe PhpOperators.declareFunc + declareCall.methodFullName shouldBe PhpOperators.declareFunc + declareCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + declareCall.code shouldBe "declare(ticks=1,encoding=\"UTF-8\")" + declareCall.lineNumber shouldBe Some(2) + + inside(declareCall.argument.l) { case List(tickAssign: Call, encodingAssign: Call) => + tickAssign.name shouldBe Operators.assignment + tickAssign.lineNumber shouldBe Some(2) + tickAssign.code shouldBe "ticks=1" + + inside(tickAssign.argument.l) { case List(ticksIdentifier: Identifier, value: Literal) => + ticksIdentifier.name shouldBe "ticks" + ticksIdentifier.code shouldBe "ticks" + + value.code shouldBe "1" + } + + encodingAssign.name shouldBe Operators.assignment + encodingAssign.lineNumber shouldBe Some(2) + encodingAssign.code shouldBe "encoding=\"UTF-8\"" + + inside(encodingAssign.argument.l) { case List(encodingIdentifier: Identifier, value: Literal) => + encodingIdentifier.name shouldBe "encoding" + encodingIdentifier.code shouldBe "encoding" + + value.code shouldBe "\"UTF-8\"" + } + } + } + + "declare calls with an empty statement list should have the correct block structure" in { + val cpg = code(""" + inside(globalMethod.body.astChildren.l) { case List(declareBlock: Block) => + declareBlock + } + } + + inside(declareBlock.astChildren.l) { case List(declareCall: Call) => + declareCall.code shouldBe "declare(ticks=1)" + } + } + + "declare calls with non-empty statement lists should have the correct block structure" in { + val cpg = code(""" + inside(globalMethod.body.astChildren.l) { case List(declareBlock: Block) => + declareBlock + } + } + + inside(declareBlock.astChildren.l) { case List(declareCall: Call, echoCall: Call) => + declareCall.code shouldBe "declare(ticks=1)" + echoCall.code shouldBe "echo \"Hello, world!\"" + } + } + + "shell_exec calls should be handled" in { + val cpg = code(" + shellCall.methodFullName shouldBe "shell_exec" + shellCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + shellCall.code shouldBe "`\"ls -la\"`" + shellCall.lineNumber shouldBe Some(2) + + inside(shellCall.argument.l) { case List(command: Literal) => + command.code shouldBe "\"ls -la\"" + } + } + } + + "unset calls should be handled" in { + val cpg = code(" + unsetCall.name shouldBe "unset" + unsetCall.methodFullName shouldBe "unset" + unsetCall.code shouldBe "unset($a, $b)" + unsetCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + unsetCall.lineNumber shouldBe Some(2) + + inside(unsetCall.argument.l) { case List(aArg: Identifier, bArg: Identifier) => + aArg.name shouldBe "a" + aArg.code shouldBe "$a" + aArg.lineNumber shouldBe Some(2) + + bArg.name shouldBe "b" + bArg.code shouldBe "$b" + bArg.lineNumber shouldBe Some(2) + } + } + } + + "global calls should handle simple and non-simple args" in { + val cpg = code(" + globalCall.name shouldBe "global" + globalCall.methodFullName shouldBe "global" + globalCall.code shouldBe "global $a, $$b" + globalCall.lineNumber shouldBe Some(2) + + inside(globalCall.argument.l) { case List(aArg: Identifier, bArg: Identifier) => + aArg.name shouldBe "a" + bArg.name shouldBe "b" + } + } + } + + "calls to builtins defined in resources/builtin_functions.txt should be handled correctly" in { + val cpg = code(" + absCall.name shouldBe "abs" + absCall.methodFullName shouldBe "abs" + absCall.code shouldBe "abs($a)" + absCall.signature shouldBe s"${Defines.UnresolvedSignature}(1)" + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/PocTest.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/PocTest.scala new file mode 100644 index 00000000..31597f99 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/PocTest.scala @@ -0,0 +1,75 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.astcreation.AstCreator.TypeConstants +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.DispatchTypes +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal} +import io.shiftleft.semanticcpg.language._ + +class PocTest extends PhpCode2CpgFixture { + + "The CPG generated for a very simple example" should { + val cpg = code( + """ + | + namespaceBlock.name shouldBe "" + case result => fail(s"expected namespaceBlock found $result") + } + } + + "have a call node for the printHello call" in { + cpg.call.nameExact("printHello").l match { + case call :: Nil => + call.lineNumber shouldBe Some(7) + + case result => fail(s"Expected printHello call got $result") + } + } + + "have the correct method ast for the printHello method" in { + cpg.method.internal.name("printHello").l match { + case method :: Nil => + val List(param) = method.parameter.l + param.name shouldBe "name" + param.code shouldBe "$name" + method.methodReturn.typeFullName shouldBe TypeConstants.Any + + val List(echoCall) = method.body.astChildren.collectAll[Call].l + echoCall.name shouldBe "echo" + echoCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + val encapsCall = echoCall.argument.l match { + case List(encapsCall: Call) => encapsCall + case result => fail(s"Expected encaps call but got $result") + } + + encapsCall.argument.l match { + case List(str1: Literal, identifier: Identifier, str2: Literal) => + str1.typeFullName shouldBe TypeConstants.String + str1.code shouldBe "\"Hello, \"" + + identifier.name shouldBe "name" + + str2.typeFullName shouldBe TypeConstants.String + str2.code shouldBe "\"\\n\"" + + case result => fail(s"Expected 3 part encaps call but got $result") + } + + case result => fail(s"Expected printHello method but got $result") + } + } + + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/ScalarTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/ScalarTests.scala new file mode 100644 index 00000000..d2fe1f26 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/ScalarTests.scala @@ -0,0 +1,87 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Literal} + +class ScalarTests extends PhpCode2CpgFixture { + "int scalars should be represented correctly" in { + val cpg = code(" + intLiteral.code shouldBe "2" + intLiteral.typeFullName shouldBe "int" + intLiteral.lineNumber shouldBe Some(2) + } + } + + "float scalars should be represented correctly" in { + val cpg = code(" + floatLiteral.code shouldBe "2.1" + floatLiteral.typeFullName shouldBe "float" + floatLiteral.lineNumber shouldBe Some(2) + } + } + + "string scalars should be represented correctly" in { + val cpg = code(" + stringLiteral.code shouldBe "\"hello\"" + stringLiteral.typeFullName shouldBe "string" + stringLiteral.lineNumber shouldBe Some(2) + } + } + + "encapsed string scalars should be represented correctly" in { + val cpg = code(" + encapsed.name shouldBe "encaps" + encapsed.typeFullName shouldBe "string" + encapsed.code shouldBe "\"hello\" . $x . $y . \" world\"" + encapsed.lineNumber shouldBe Some(2) + + inside(encapsed.astChildren.l) { case List(hello: Literal, x: Identifier, y: Identifier, world: Literal) => + hello.code shouldBe "\"hello\"" + hello.typeFullName shouldBe "string" + hello.lineNumber shouldBe Some(2) + + world.code shouldBe "\" world\"" + world.typeFullName shouldBe "string" + world.lineNumber shouldBe Some(2) + + x.name shouldBe "x" + x.lineNumber shouldBe Some(2) + + y.name shouldBe "y" + y.lineNumber shouldBe Some(2) + } + } + } + + "booleans should be represented as literals" in { + val cpg = code(" + trueBool.code shouldBe "true" + trueBool.lineNumber shouldBe Some(2) + trueBool.typeFullName shouldBe "bool" + + falseBool.code shouldBe "false" + falseBool.lineNumber shouldBe Some(2) + falseBool.typeFullName shouldBe "bool" + } + } + "null should be represented as a literal" in { + val cpg = code(" + nullLiteral.code shouldBe "NULL" + nullLiteral.lineNumber shouldBe Some(2) + nullLiteral.typeFullName shouldBe "null" + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/TypeDeclTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/TypeDeclTests.scala new file mode 100644 index 00000000..0b5d5c79 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/TypeDeclTests.scala @@ -0,0 +1,286 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.appthreat.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.{ModifierTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal, Local, Member, Method} +import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.Block +import io.shiftleft.codepropertygraph.generated.nodes.MethodRef +import io.shiftleft.codepropertygraph.generated.nodes.TypeRef + +class TypeDeclTests extends PhpCode2CpgFixture { + + "typedecl nodes for empty classes should have the correct basic properties set" in { + val cpg = code(""" + typeDecl.fullName shouldBe "A" + typeDecl.lineNumber shouldBe Some(2) + typeDecl.code shouldBe "class A extends B implements C, D" + } + } + + "class methods should be created correctly" in { + val cpg = code(""" + fooMethod.fullName shouldBe s"Foo->foo" + fooMethod.signature shouldBe s"${Defines.UnresolvedSignature}(1)" + fooMethod.modifier.map(_.modifierType).toSet shouldBe Set(ModifierTypes.FINAL, ModifierTypes.PUBLIC) + fooMethod.methodReturn.typeFullName shouldBe "int" + inside(fooMethod.parameter.l) { case List(thisParam, xParam) => + thisParam.name shouldBe "this" + thisParam.code shouldBe "this" + thisParam.dynamicTypeHintFullName should contain("Foo") + thisParam.typeFullName shouldBe "Foo" + thisParam.index shouldBe 0 + + xParam.code shouldBe "$x" + xParam.typeFullName shouldBe "int" + xParam.index shouldBe 1 + } + } + } + + "constructors using the class name should be represented with the correct init method" in { + val cpg = code(""" + tmpLocal.name shouldBe "tmp0" + tmpLocal.code shouldBe "$tmp0" + + constructorBlock.lineNumber shouldBe Some(3) + + inside(constructorBlock.astChildren.l) { case List(allocAssign: Call, initCall: Call, tmpVar: Identifier) => + allocAssign.methodFullName shouldBe Operators.assignment + inside(allocAssign.astChildren.l) { case List(tmpIdentifier: Identifier, allocCall: Call) => + tmpIdentifier.name shouldBe "tmp0" + tmpIdentifier.code shouldBe "$tmp0" + tmpIdentifier._localViaRefOut should contain(tmpLocal) + + allocCall.name shouldBe Operators.alloc + allocCall.methodFullName shouldBe Operators.alloc + allocCall.lineNumber shouldBe Some(3) + allocCall.code shouldBe "Foo.()" + } + + initCall.name shouldBe "__construct" + initCall.methodFullName shouldBe s"Foo->__construct" + initCall.signature shouldBe s"${Defines.UnresolvedSignature}(1)" + initCall.code shouldBe "Foo->__construct(42)" + inside(initCall.argument.l) { case List(tmpIdentifier: Identifier, literal: Literal) => + tmpIdentifier.name shouldBe "tmp0" + tmpIdentifier.code shouldBe "$tmp0" + tmpIdentifier.argumentIndex shouldBe 0 + tmpIdentifier._localViaRefOut should contain(tmpLocal) + literal.code shouldBe "42" + literal.argumentIndex shouldBe 1 + } + } + } + } + + "constructors using expressions for the class name should have the correct alloc receiver" in { + val cpg = code(""" + alloc.name shouldBe Operators.alloc + alloc.methodFullName shouldBe Operators.alloc + alloc.code shouldBe "$x.()" + inside(alloc.argument(0).start.l) { case List(xIdentifier: Identifier) => + xIdentifier.name shouldBe "x" + xIdentifier.code shouldBe "$x" + } + } + } + + "interfaces not extending other interfaces should be created correctly" in { + val cpg = code(""" + fooDecl.fullName shouldBe "Foo" + fooDecl.code shouldBe "interface Foo" + fooDecl.inheritsFromTypeFullName.isEmpty shouldBe true + + inside(fooDecl.astChildren.l) { case List(fooMethod: Method) => + fooMethod.name shouldBe "foo" + fooMethod.fullName shouldBe s"Foo->foo" + fooMethod.signature shouldBe s"${Defines.UnresolvedSignature}(0)" + } + } + } + + "interfaces should be able to extend multiple other interfaces" in { + val cpg = code(""" + fooDecl.fullName shouldBe "Foo" + fooDecl.code shouldBe "interface Foo extends Bar, Baz" + fooDecl.inheritsFromTypeFullName should contain theSameElementsAs List("Bar", "Baz") + } + } + + "traits should have the correct code fields" in { + val cpg = code(""" + fooDecl.fullName shouldBe "Foo" + fooDecl.code shouldBe "trait Foo" + fooDecl.inheritsFromTypeFullName.isEmpty shouldBe true + + inside(fooDecl.astChildren.l) { case List(fooMethod: Method) => + fooMethod.name shouldBe "foo" + fooMethod.fullName shouldBe s"Foo->foo" + fooMethod.signature shouldBe s"${Defines.UnresolvedSignature}(0)" + } + } + } + + "enums with cases without values should have the correct fields" in { + val cpg = code(""" + fooDecl.fullName shouldBe "Foo" + fooDecl.code shouldBe "enum Foo" + + inside(fooDecl.astChildren.l) { case List(aMember: Member, bMember: Member) => + aMember.name shouldBe "A" + aMember.code shouldBe "case A" + aMember.lineNumber shouldBe Some(3) + + bMember.name shouldBe "B" + bMember.code shouldBe "case B" + bMember.lineNumber shouldBe Some(4) + } + } + } + + "enums with cases with values should have the correct initializers" in { + val cpg = code( + """ + fooDecl.fullName shouldBe "Foo" + fooDecl.code shouldBe "enum Foo" + + inside(fooDecl.member.l) { case List(aMember: Member, bMember: Member) => + aMember.name shouldBe "A" + aMember.code shouldBe "case A" + aMember.lineNumber shouldBe Some(3) + + bMember.name shouldBe "B" + bMember.code shouldBe "case B" + bMember.lineNumber shouldBe Some(4) + } + + inside(fooDecl.method.l) { case List(clinitMethod: Method) => + clinitMethod.name shouldBe Defines.StaticInitMethodName + clinitMethod.fullName shouldBe s"Foo::${Defines.StaticInitMethodName}" + clinitMethod.signature shouldBe "void()" + clinitMethod.filename shouldBe "foo.php" + clinitMethod.file.name.l shouldBe List("foo.php") + + inside(clinitMethod.body.astChildren.l) { case List(aAssign: Call, bAssign: Call) => + aAssign.code shouldBe "A = \"A\"" + inside(aAssign.astChildren.l) { case List(aIdentifier: Identifier, aLiteral: Literal) => + aIdentifier.name shouldBe "A" + aIdentifier.code shouldBe "A" + + aLiteral.code shouldBe "\"A\"" + } + + bAssign.code shouldBe "B = \"B\"" + inside(bAssign.astChildren.l) { case List(bIdentifier: Identifier, bLiteral: Literal) => + bIdentifier.name shouldBe "B" + bIdentifier.code shouldBe "B" + + bLiteral.code shouldBe "\"B\"" + } + } + } + } + } + + "the global type decl should have the correct name" in { + val cpg = code("").fullName.l shouldBe List("foo.php:") + } + + "class magic constants" when { + "called on a class name should give the fully qualified class name" in { + val cpg = code(""" + fooRef.typeFullName shouldBe "foo\\Foo" + fooRef.lineNumber shouldBe Some(6) + } + } + + "called on an object should give the fully qualified type name" in { + val cpg = code(""" + // TODO The typeFullName here is missing, even though we should get it. Fix with types in general. + // fooRef.typeFullName shouldBe "foo\\Foo" + fooRef.lineNumber shouldBe Some(6) + } + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/TypeNodeTests.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/TypeNodeTests.scala new file mode 100644 index 00000000..23299cec --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/querying/TypeNodeTests.scala @@ -0,0 +1,38 @@ +package io.appthreat.php2atom.querying + +import io.appthreat.php2atom.testfixtures.PhpCode2CpgFixture +import io.appthreat.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.{ModifierTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal, Local, Member, Method} +import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.Block + +class TypeNodeTests extends PhpCode2CpgFixture { + "TypeDecls with inheritsFrom types" should { + val cpg = code(""" + importStmt.code shouldBe "use A\\B" + importStmt.importedEntity should contain("A\\B") + importStmt.importedAs.isEmpty shouldBe true + } + } + + "normal use statements including multiple namespaces should be enclosed in a block" in { + val cpg = code(" + aImport.code shouldBe "use A" + aImport.importedEntity should contain("A") + aImport.importedAs.isEmpty shouldBe true + + bImport.code shouldBe "use B" + bImport.importedEntity should contain("B") + bImport.importedAs.isEmpty shouldBe true + } + } + + "use statements with aliases should be correctly represented" in { + val cpg = code(" + importStmt.code shouldBe "use A\\B as C" + importStmt.importedEntity should contain("A\\B") + importStmt.importedAs should contain("C") + } + } + + "function uses should have the correct code field" in { + val cpg = code(" + importStmt.code shouldBe "use function foo\\bar" + importStmt.importedEntity should contain("foo\\bar") + importStmt.importedAs.isEmpty shouldBe true + } + } + + "const uses should have the correct code field" in { + val cpg = code(" + importStmt.code shouldBe "use const foo\\BAR" + importStmt.importedEntity should contain("foo\\BAR") + importStmt.importedAs.isEmpty shouldBe true + } + } + + "group uses should have the correct names for all elements in the group" in { + val cpg = code(" + aImport.code shouldBe "use A\\B\\C" + aImport.importedEntity should contain("A\\B\\C") + aImport.importedAs.isEmpty shouldBe true + + bImport.code shouldBe "use A\\D" + bImport.importedEntity should contain("A\\D") + bImport.importedAs.isEmpty shouldBe true + } + } +} diff --git a/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/testfixtures/PhpCode2CpgFixture.scala b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/testfixtures/PhpCode2CpgFixture.scala new file mode 100644 index 00000000..e86eb7e0 --- /dev/null +++ b/platform/frontends/php2atom/src/test/scala/io/appthreat/php2atom/testfixtures/PhpCode2CpgFixture.scala @@ -0,0 +1,44 @@ +package io.appthreat.php2atom.testfixtures + +import io.appthreat.dataflowengineoss.queryengine.EngineContext +import io.appthreat.php2atom.{Config, Php2Atom} +import io.appthreat.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend} +import io.appthreat.x2cpg.passes.frontend.XTypeRecoveryConfig +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} + +import java.io.File +import io.appthreat.x2cpg.testfixtures.TestCpg +import io.appthreat.x2cpg.X2Cpg +import io.shiftleft.semanticcpg.layers.LayerCreatorContext +import io.appthreat.dataflowengineoss.layers.dataflows.OssDataFlowOptions +import io.appthreat.dataflowengineoss.layers.dataflows.OssDataFlow +import io.appthreat.php2atom.passes.PhpSetKnownTypesPass + +trait PhpFrontend extends LanguageFrontend { + override val fileSuffix: String = ".php" + + override def execute(sourceCodeFile: File): Cpg = { + implicit val defaultConfig: Config = getConfig().map(_.asInstanceOf[Config]).getOrElse(Config()) + new Php2Atom().createCpg(sourceCodeFile.getAbsolutePath).get + } +} + +class PhpTestCpg(runOssDataflow: Boolean) extends TestCpg with PhpFrontend { + + override protected def applyPasses(): Unit = { + X2Cpg.applyDefaultOverlays(this) + if (runOssDataflow) { + val context = new LayerCreatorContext(this) + val options = new OssDataFlowOptions() + new OssDataFlow(options).run(context) + } + Php2Atom.postProcessingPasses(this).foreach(_.createAndApply()) + } +} + +class PhpCode2CpgFixture(runOssDataflow: Boolean = false) + extends Code2CpgFixture(() => new PhpTestCpg(runOssDataflow)) { + implicit val resolver: ICallResolver = NoResolve + implicit lazy val engineContext: EngineContext = EngineContext() +} diff --git a/platform/frontends/pysrc2cpg/README.md b/platform/frontends/pysrc2cpg/README.md deleted file mode 100644 index 6940736e..00000000 --- a/platform/frontends/pysrc2cpg/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Stage and run: - - Build and assemble pysrc2cpg for local execution: sbt pysrc2cpg/stage - - Run: ./pysrc2cpg.sh -o - -Shortcomings of Python CPG representation: - - No named parameter support - - Incorrect instance argument for call like x.func. - See source code comment. - - No handling of __getattr__, __setattr__, etc. diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/AstCreatorBase.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/AstCreatorBase.scala index e26454c5..42c611d0 100644 --- a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/AstCreatorBase.scala +++ b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/AstCreatorBase.scala @@ -4,12 +4,13 @@ import io.appthreat.x2cpg.passes.frontend.MetaDataPass import io.appthreat.x2cpg.utils.NodeBuilders.newMethodReturnNode import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, ModifierTypes} +import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import overflowdb.BatchedUpdate.DiffGraphBuilder abstract class AstCreatorBase(filename: String)(implicit withSchemaValidation: ValidationMode): val diffGraph: DiffGraphBuilder = new DiffGraphBuilder - + private val closureKeyPool = new IntervalKeyPool(first = 0, last = Long.MaxValue) def createAst(): DiffGraphBuilder /** Create a global namespace block for the given `filename` @@ -326,4 +327,6 @@ abstract class AstCreatorBase(filename: String)(implicit withSchemaValidation: V */ def absolutePath(filename: String): String = better.files.File(filename).path.toAbsolutePath.normalize().toString + + def nextClosureName(): String = s"${Defines.ClosurePrefix}${closureKeyPool.next}" end AstCreatorBase diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/Defines.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/Defines.scala index e628afc5..3bfd95ad 100644 --- a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/Defines.scala +++ b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/Defines.scala @@ -28,4 +28,5 @@ object Defines: val LeftAngularBracket = "<" val Unknown = "" + val ClosurePrefix = "" end Defines diff --git a/project/Projects.scala b/project/Projects.scala index e5b31b36..5c7da625 100644 --- a/project/Projects.scala +++ b/project/Projects.scala @@ -15,4 +15,5 @@ object Projects { lazy val jssrc2cpg = project.in(frontendsRoot / "jssrc2cpg") lazy val javasrc2cpg = project.in(frontendsRoot / "javasrc2cpg") lazy val jimple2cpg = project.in(frontendsRoot / "jimple2cpg") + lazy val php2atom = project.in(frontendsRoot / "php2atom") } diff --git a/pyproject.toml b/pyproject.toml index 2d722c45..d81d11da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appthreat-chen" -version = "1.1.2" +version = "1.1.3" description = "Code Hierarchy Exploration Net (chen)" authors = ["Team AppThreat "] license = "Apache-2.0"