Merge pull request #3417 from aidan-harding:language-detection

[core] Support forcing a specific language from the command-line #3417

docs/pages/pmd/userdocs/cli_reference.md

@@ -79,6 +79,17 @@ The tool comes with a rather extensive help text, simply running with `-help`!
                description="Path to file containing a comma delimited list of files to analyze.
                             If this is given, then you don't need to provide `-dir`."
     %}
+    {% include custom/cli_option_row.html options="-force-language"
+               option_arg="lang"
+               description="Force a language to be used for all input files, irrespective of
+                            filenames. When using this option, the automatic language selection
+                            by extension is disabled and all files are tried to be parsed with
+                            the given language `<lang>`. Parsing errors are ignored and unparsable files
+                            are skipped.
+                            
+                            <p>This option allows to use the xml language for files, that don't
+                            use xml as extension. See [example](#analyze-other-xml-formats) below.</p>"
+    %}
     {% include custom/cli_option_row.html options="-ignorelist"
                option_arg="filepath"
                description="Path to file containing a comma delimited list of files to ignore.
@@ -202,3 +213,26 @@ Example:
 PMD comes with many different renderers.
 All formats are described at [PMD Report formats](pmd_userdocs_report_formats.html)
 
+## Examples
+
+### Analyze other xml formats
+
+If your xml language doesn't use `xml` as file extension, you can still use PMD with `-force-language`:
+
+```
+$ ./run.sh pmd -d /home/me/src/xml-file.ext -f text -R ruleset.xml -force-language xml
+```
+
+You can also specify a directory instead of a single file. Then all files are analyzed. In that case,
+parse errors are suppressed in order to reduce irrelevant noise:
+
+```
+$ ./run.sh pmd -d /home/me/src/ -f text -R ruleset.xml -force-language xml
+```
+
+Alternatively, you can create a filelist to only analyze files with a given extension:
+
+```
+$ find /home/me/src -name "*.ext" > /home/me/src/filelist.txt
+$ ./run.sh pmd -filelist /home/me/src/filelist.txt -f text -R ruleset.xml -force-language xml
+```

docs/pages/release_notes.md

@@ -110,6 +110,7 @@ This release ships with 3 new Java rules.
     *   [#3329](https://github.com/pmd/pmd/issues/3329): \[apex] ApexCRUDViolation doesn't report SOQL for loops
 *   core
     *   [#1603](https://github.com/pmd/pmd/issues/1603): \[core] Language version comparison
+    *   [#2133](https://github.com/pmd/pmd/issues/2133): \[xml] Allow to check Salesforce XML Metadata using XPath rules
     *   [#3377](https://github.com/pmd/pmd/issues/3377): \[core] NPE when specifying report file in current directory in PMD CLI
     *   [#3387](https://github.com/pmd/pmd/issues/3387): \[core] CPD should avoid unnecessary copies when running with --skip-lexical-errors
 *   java-bestpractices
@@ -123,6 +124,16 @@ This release ships with 3 new Java rules.
 
 ### API Changes
 
+#### PMD CLI
+
+*   PMD has a new CLI option `-force-language`. With that a language can be forced to be used for all input files,
+    irrespective of filenames. When using this option, the automatic language selection by extension is disabled
+    and all files are tried to be parsed with the given language. Parsing errors are ignored and unparsable files
+    are skipped.
+
+    This option allows to use the xml language for files, that don't use xml as extension.
+    See also the examples on [PMD CLI reference](pmd_userdocs_cli_reference.html#analyze-other-xml-formats).
+
 #### Experimental APIs
 
 *   The AST types and APIs around Sealed Classes are not experimental anymore:
@@ -145,6 +156,7 @@ You can identify them with the `@InternalApi` annotation. You'll also get a depr
 *   [#3373](https://github.com/pmd/pmd/pull/3373): \[apex] Add ApexCRUDViolation support for database class, inline no-arg object construction DML and inline list initialization DML - [Jonathan Wiesel](https://github.com/jonathanwiesel)
 *   [#3385](https://github.com/pmd/pmd/pull/3385): \[core] CPD: Optimize --skip-lexical-errors option - [Woongsik Choi](https://github.com/woongsikchoi)
 *   [#3388](https://github.com/pmd/pmd/pull/3388): \[doc] Add Code Inspector in the list of tools - [Julien Delange](https://github.com/juli1)
+*   [#3417](https://github.com/pmd/pmd/pull/3417): \[core] Support forcing a specific language from the command-line - [Aidan Harding](https://github.com/aidan-harding)
 
 {% endtocmaker %}
 

pmd-core/src/main/java/net/sourceforge/pmd/PMD.java

@@ -5,6 +5,7 @@
 package net.sourceforge.pmd;
 
 import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
@@ -417,7 +418,7 @@ public static List<DataSource> getApplicableFiles(PMDConfiguration configuration
 
     private static List<DataSource> internalGetApplicableFiles(PMDConfiguration configuration,
                                                                Set<Language> languages) {
-        LanguageFilenameFilter fileSelector = new LanguageFilenameFilter(languages);
+        FilenameFilter fileSelector = configuration.isForceLanguageVersion() ? new AcceptAllFilenames() : new LanguageFilenameFilter(languages);
         List<DataSource> files = new ArrayList<>();
 
         if (null != configuration.getInputPaths()) {
@@ -636,4 +637,11 @@ public int toInt() {
         }
 
     }
+
+    private static class AcceptAllFilenames implements FilenameFilter {
+        @Override
+        public boolean accept(File dir, String name) {
+            return true;
+        }
+    }
 }

pmd-core/src/main/java/net/sourceforge/pmd/PMDConfiguration.java

@@ -88,6 +88,7 @@ public class PMDConfiguration extends AbstractConfiguration {
     private int threads = Runtime.getRuntime().availableProcessors();
     private ClassLoader classLoader = getClass().getClassLoader();
     private LanguageVersionDiscoverer languageVersionDiscoverer = new LanguageVersionDiscoverer();
+    private LanguageVersion forceLanguageVersion;
 
     // Rule and source file options
     private String ruleSets;
@@ -210,6 +211,35 @@ public LanguageVersionDiscoverer getLanguageVersionDiscoverer() {
         return languageVersionDiscoverer;
     }
 
+    /**
+     * Get the LanguageVersion specified by the force-language parameter. This overrides detection based on file
+     * extensions
+     *
+     * @return The LanguageVersion.
+     */
+    public LanguageVersion getForceLanguageVersion() {
+        return forceLanguageVersion;
+    }
+
+    /**
+     * Is the force-language parameter set to anything?
+     *
+     * @return true if ${@link #getForceLanguageVersion()} is not null
+     */
+    public boolean isForceLanguageVersion() {
+        return forceLanguageVersion != null;
+    }
+
+    /**
+     * Set the LanguageVersion specified by the force-language parameter. This overrides detection based on file
+     * extensions
+     *
+     * @param forceLanguageVersion the language version
+     */
+    public void setForceLanguageVersion(LanguageVersion forceLanguageVersion) {
+        this.forceLanguageVersion = forceLanguageVersion;
+    }
+
     /**
      * Set the given LanguageVersion as the current default for it's Language.
      *

pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java

@@ -10,6 +10,8 @@
 import java.io.Reader;
 import java.util.Collections;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import net.sourceforge.pmd.annotation.InternalApi;
 import net.sourceforge.pmd.benchmark.TimeTracker;
@@ -31,6 +33,8 @@
 @InternalApi
 public class SourceCodeProcessor {
 
+    private static final Logger LOG = Logger.getLogger(SourceCodeProcessor.class.getName());
+
     private final PMDConfiguration configuration;
 
     public SourceCodeProcessor(PMDConfiguration configuration) {
@@ -113,7 +117,11 @@ private void processSourceCodeWithoutCache(final Reader sourceCode, final RuleSe
             processSource(sourceCode, ruleSets, ctx);
         } catch (ParseException pe) {
             configuration.getAnalysisCache().analysisFailed(ctx.getSourceCodeFile());
-            throw new PMDException("Error while parsing " + ctx.getSourceCodeFile(), pe);
+            if (configuration.isForceLanguageVersion()) {
+                LOG.log(Level.FINE, "Error while parsing " + ctx.getSourceCodeFile(), pe);
+            } else {
+                throw new PMDException("Error while parsing " + ctx.getSourceCodeFile(), pe);
+            }
         } catch (Exception e) {
             configuration.getAnalysisCache().analysisFailed(ctx.getSourceCodeFile());
             throw new PMDException("Error while processing " + ctx.getSourceCodeFile(), e);
@@ -201,9 +209,18 @@ private void processSource(Reader sourceCode, RuleSets ruleSets, RuleContext ctx
     }
 
     private void determineLanguage(RuleContext ctx) {
-        // If LanguageVersion of the source file is not known, make a
-        // determination
-        if (ctx.getLanguageVersion() == null) {
+        if (ctx.getLanguageVersion() != null) {
+            // we already have a language
+            return;
+        }
+
+        // If LanguageVersion of the source file is not known, make a determination
+        LanguageVersion forceLanguage = configuration.getForceLanguageVersion();
+        if (forceLanguage != null) {
+            // use force language if given
+            ctx.setLanguageVersion(forceLanguage);
+        } else {
+            // otherwise determine by file extension
             LanguageVersion languageVersion = configuration.getLanguageVersionOfFile(ctx.getSourceCodeFilename());
             ctx.setLanguageVersion(languageVersion);
         }

pmd-core/src/main/java/net/sourceforge/pmd/cli/PMDParameters.java

@@ -101,6 +101,9 @@ public class PMDParameters {
     @Parameter(names = { "-language", "-l" }, description = "Specify a language PMD should use.")
     private String language = null;
 
+    @Parameter(names = "-force-language", description = "Force a language to be used for all input files, irrespective of filenames.")
+    private String forceLanguage = null;
+
     @Parameter(names = "-auxclasspath",
             description = "Specifies the classpath for libraries used by the source code. "
                     + "This is used by the type resolution. The platform specific path delimiter "
@@ -214,11 +217,18 @@ public PMDConfiguration toConfiguration() {
         configuration.setAnalysisCacheLocation(this.cacheLocation);
         configuration.setIgnoreIncrementalAnalysis(this.isIgnoreIncrementalAnalysis());
 
+        LanguageVersion forceLangVersion = LanguageRegistry
+                .findLanguageVersionByTerseName(this.getForceLanguage());
+        if (forceLangVersion != null) {
+            configuration.setForceLanguageVersion(forceLangVersion);
+        }
+
         LanguageVersion languageVersion = LanguageRegistry
                 .findLanguageVersionByTerseName(this.getLanguage() + ' ' + this.getVersion());
         if (languageVersion != null) {
             configuration.getLanguageVersionDiscoverer().setDefaultLanguageVersion(languageVersion);
         }
+
         try {
             configuration.prependClasspath(this.getAuxclasspath());
         } catch (IOException e) {
@@ -305,6 +315,10 @@ public String getLanguage() {
         return language != null ? language : LanguageRegistry.getDefaultLanguage().getTerseName();
     }
 
+    public String getForceLanguage() {
+        return forceLanguage != null ? forceLanguage : "";
+    }
+
     public String getAuxclasspath() {
         return auxclasspath;
     }

pmd-core/src/main/java/net/sourceforge/pmd/util/FileUtil.java

@@ -174,7 +174,7 @@ public static boolean findPatternInFile(final File file, final String pattern) {
     /**
      * Reads the file, which contains the filelist. This is used for the
      * command line arguments --filelist/-filelist for both PMD and CPD.
-     * The separator in the filelist is a command and/or newlines.
+     * The separator in the filelist is a comma and/or newlines.
      *
      * @param filelist the file which contains the list of path names
      * @return a comma-separated list of file paths

pmd-core/src/test/java/net/sourceforge/pmd/lang/LanguageParameterTest.java

@@ -0,0 +1,26 @@
+/**
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import net.sourceforge.pmd.cli.PMDCommandLineInterface;
+import net.sourceforge.pmd.cli.PMDParameters;
+
+public class LanguageParameterTest {
+
+    /** Test that language parameters from the CLI are correctly passed through to the PMDConfiguration. Although this is a
+     * CLI test, it resides here to take advantage of {@link net.sourceforge.pmd.lang.DummyLanguageModule}
+     */
+    @Test
+    public void testLanguageFromCliToConfiguration() {
+        PMDParameters params = new PMDParameters();
+        String[] args = { "-d", "source_folder", "-f", "ideaj", "-P", "sourcePath=/home/user/source/", "-R", "java-empty", "-force-language", "dummy"};
+        PMDCommandLineInterface.extractParameters(params, args, "PMD");
+
+        Assert.assertEquals(new DummyLanguageModule().getDefaultVersion().getName(), params.toConfiguration().getForceLanguageVersion().getName());
+    }
+}

pmd-xml/src/test/java/net/sourceforge/pmd/lang/xml/XmlCliTest.java

@@ -0,0 +1,66 @@
+/*
+ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+ */
+
+package net.sourceforge.pmd.lang.xml;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import net.sourceforge.pmd.cli.BaseCLITest;
+
+public class XmlCliTest extends BaseCLITest {
+    private static final String BASE_DIR = "src/test/resources/net/sourceforge/pmd/lang/xml/cli-tests/sampleproject";
+    private static final String RULE_MESSAGE = "A tags are not allowed";
+
+    private String[] createArgs(String directory, String ... args) {
+        List<String> arguments = new ArrayList<>();
+        arguments.add("-f");
+        arguments.add("text");
+        arguments.add("-no-cache");
+        arguments.add("-R");
+        arguments.add(BASE_DIR + "/ruleset.xml");
+        arguments.add("-d");
+        arguments.add(BASE_DIR + directory);
+        arguments.addAll(Arrays.asList(args));
+        return arguments.toArray(new String[0]);
+    }
+
+    @Test
+    public void analyzeSingleXmlWithoutForceLanguage() {
+        String resultFilename = runTest(createArgs("/src/file1.ext"), "analyzeSingleXmlWithoutForceLanguage", 0);
+        assertRuleMessage(0, resultFilename);
+    }
+
+    @Test
+    public void analyzeSingleXmlWithForceLanguage() {
+        String resultFilename = runTest(createArgs("/src/file1.ext", "-force-language", "xml"),
+            "analyzeSingleXmlWithForceLanguage", 4);
+        assertRuleMessage(1, resultFilename);
+    }
+
+    @Test
+    public void analyzeDirectoryWithForceLanguage() {
+        String resultFilename = runTest(createArgs("/src/", "-force-language", "xml"),
+            "analyzeDirectoryWithForceLanguage", 4);
+        assertRuleMessage(3, resultFilename);
+    }
+
+    private void assertRuleMessage(int expectedCount, String resultFilename) {
+        try {
+            String result = FileUtils.readFileToString(new File(resultFilename), StandardCharsets.UTF_8);
+            Assert.assertEquals(expectedCount, StringUtils.countMatches(result, RULE_MESSAGE));
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+    }
+}

pmd-xml/src/test/resources/net/sourceforge/pmd/lang/xml/cli-tests/sampleproject/ruleset.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+
+<ruleset name="sample"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
+
+    <description>
+Sample
+    </description>
+
+    <rule name="A"
+          language="xml"
+          message="A tags are not allowed"
+          class="net.sourceforge.pmd.lang.rule.XPathRule">
+        <description>
+A tags are not allowed
+        </description>
+        <priority>3</priority>
+        <properties>
+            <property name="version" value="2.0"/>
+            <property name="xpath">
+                <value>
+<![CDATA[
+//a
+]]>
+                </value>
+            </property>
+        </properties>
+    </rule>
+</ruleset>

pmd-xml/src/test/resources/net/sourceforge/pmd/lang/xml/cli-tests/sampleproject/src/file1.ext

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- BSD-style license; for more info see http://pmd.sourceforge.net/license.html -->
+<file>
+  <a></a>
+</file>

pmd-xml/src/test/resources/net/sourceforge/pmd/lang/xml/cli-tests/sampleproject/src/file2.ext

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- BSD-style license; for more info see http://pmd.sourceforge.net/license.html -->
+<file>
+  <a></a>
+  <a></a>
+</file>

pmd-xml/src/test/resources/net/sourceforge/pmd/lang/xml/cli-tests/sampleproject/src/file3.txt

@@ -0,0 +1,3 @@
+BSD-style license; for more info see http://pmd.sourceforge.net/license.html
+
+Other file that is not a xml file.